| Key | Value | Timestamp | Execution Date | Task Id | Dag Id | ||
|---|---|---|---|---|---|---|---|
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T17:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T17:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T17:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T17:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T17:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T17:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "18", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "17", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "18", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "17", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "18", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "17", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-08T17:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-08T17:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-08 18:00:13.294481+00:00 | 2024-08-08 17:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-3R8K5P7QIDQBY | 2024-08-08 18:00:34.446885+00:00 | 2024-08-08 17:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T17:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-08 18:05:08.299837+00:00 | 2024-08-08 17:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-4KQLGW3CIAW1 | 2024-08-08 18:05:28.462927+00:00 | 2024-08-08 17:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T18:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T18:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T18:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T18:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T18:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T18:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "19", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "18", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "19", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "18", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "19", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "18", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-08T18:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-08T18:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-08 19:00:24.079879+00:00 | 2024-08-08 18:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-3ERW51Y7HH8DO | 2024-08-08 19:00:44.462855+00:00 | 2024-08-08 18:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T18:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-08 19:05:27.763381+00:00 | 2024-08-08 18:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-2FZY3Z68CW4R0 | 2024-08-08 19:05:56.342668+00:00 | 2024-08-08 18:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T19:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T19:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T19:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T19:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T19:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T19:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "20", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "19", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "20", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "19", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "20", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "19", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-08T19:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-08T19:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-08 20:00:16.194905+00:00 | 2024-08-08 19:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-1271LV5EJXOYF | 2024-08-08 20:00:54.785877+00:00 | 2024-08-08 19:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T19:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-08 20:05:17.656634+00:00 | 2024-08-08 19:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1M03VDD313OMM | 2024-08-08 20:05:39.857724+00:00 | 2024-08-08 19:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T20:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T20:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T20:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T20:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T20:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T20:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "21", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "20", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "21", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "20", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "21", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "20", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-08T20:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-08T20:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-08 21:00:15.326674+00:00 | 2024-08-08 20:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-28LT3XNJVNP63 | 2024-08-08 21:00:34.314210+00:00 | 2024-08-08 20:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T20:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-08 21:05:06.698541+00:00 | 2024-08-08 20:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-2STQ3IZK7WV89 | 2024-08-08 21:05:26.311110+00:00 | 2024-08-08 20:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T21:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T21:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T21:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T21:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T21:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T21:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "22", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "21", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "22", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "21", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "22", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "21", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-08T21:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-08T21:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-08 22:00:09.745986+00:00 | 2024-08-08 21:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-9CYNWF0J4WU0 | 2024-08-08 22:00:28.412774+00:00 | 2024-08-08 21:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T21:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-08 22:05:06.421626+00:00 | 2024-08-08 21:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-23YYBZUQEWWUA | 2024-08-08 22:05:28.858924+00:00 | 2024-08-08 21:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T22:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T22:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T22:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T22:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T22:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T22:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "23", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "22", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "23", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "22", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "23", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "22", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-08T22:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-08T22:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-08 23:00:23.325195+00:00 | 2024-08-08 22:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-campaign-delta-past-delete-job", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "CampaignDeltaPastDeleteJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-07T23:00:00+00:00", "--class", "com.viooh.booking.delta.CampaignDeltaPastDeleteMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/pandora-campaign-delta/1.22.0/pandora-campaign-delta-1.22.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-campaign-delta-past-delete-job"}], "Configurations": []} | 2024-08-08 23:00:27.837211+00:00 | 2024-08-07 23:00:00+00:00 | create_emr_steps | campaign-delta-past-delete-job_v002 | ||
| return_value | j-3BSQL7SDVUDAG | 2024-08-08 23:00:42.238589+00:00 | 2024-08-08 22:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-30BQM6EABJ5LB | 2024-08-08 23:00:47.887742+00:00 | 2024-08-07 23:00:00+00:00 | create_cluster_and_add_emr_steps | campaign-delta-past-delete-job_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T22:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-08 23:05:12.653566+00:00 | 2024-08-08 22:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1KXWHH03PQ204 | 2024-08-08 23:05:34.502958+00:00 | 2024-08-08 22:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T23:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T23:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T23:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T23:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T23:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T23:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "0", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "23", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "0", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "23", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "0", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-08", "23", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-08T23:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-08T23:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-09 00:00:25.960048+00:00 | 2024-08-08 23:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-1BU2HILRLN9GC | 2024-08-09 00:00:46.886096+00:00 | 2024-08-08 23:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T23:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-09 00:05:16.703361+00:00 | 2024-08-08 23:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-3RS109TTGCFDD | 2024-08-09 00:05:36.060445+00:00 | 2024-08-08 23:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T00:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T00:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T00:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T00:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T00:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T00:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "1", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "0", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "1", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "0", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "1", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "0", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-09T00:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-09T00:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-09 01:00:28.780382+00:00 | 2024-08-09 00:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-3SMSNHDJ5QUYK | 2024-08-09 01:00:51.055793+00:00 | 2024-08-09 00:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T00:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-09 01:05:14.011039+00:00 | 2024-08-09 00:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-2F810ZCNAMSIA | 2024-08-09 01:05:35.143059+00:00 | 2024-08-09 00:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-booking-by-hour-model-transform", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.2xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}, {"InstanceType": "m5.4xlarge", "EbsConfiguration": {"EbsBlockDeviceConfigs": [{"VolumeSpecification": {"VolumeType": "gp2", "SizeInGB": 50}, "VolumesPerInstance": 1}], "EbsOptimized": true}}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "BookingByHour-ModelTransform", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=20g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T01:30:00+00:00", "--class", "com.viooh.transform.BookingModelTransform", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/0.0.0/job.yaml", "s3://viooh-spark-artifacts-lab-cn/releases/uk/co/viooh/pandora-model-transform/1.7.0/pandora-model-transform-1.7.0.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-booking-by-hour-model-transform"}], "Configurations": []} | 2024-08-09 01:30:25.802513+00:00 | 2024-08-08 01:30:00+00:00 | create_emr_steps | booking-by-hour-model-transform_v002 | ||
| return_value | j-215OXKQ7VDOJA | 2024-08-09 01:30:45.635607+00:00 | 2024-08-08 01:30:00+00:00 | create_cluster_and_add_emr_steps | booking-by-hour-model-transform_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T01:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T01:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T01:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T01:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T01:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T01:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "2", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "1", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "2", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "1", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "2", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "1", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-09T01:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-09T01:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-09 02:02:22.401667+00:00 | 2024-08-09 01:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-PXO5KPO112MA | 2024-08-09 02:02:44.588223+00:00 | 2024-08-09 01:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T01:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-09 02:10:37.523908+00:00 | 2024-08-09 01:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1NS76HSBX4TS2 | 2024-08-09 02:10:49.253695+00:00 | 2024-08-09 01:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T02:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T02:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T02:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T02:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T02:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T02:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "3", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "2", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "3", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "2", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "3", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "2", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-09T02:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-09T02:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-09 03:00:22.744764+00:00 | 2024-08-09 02:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-1DGE91OLP8RQR | 2024-08-09 03:00:59.865817+00:00 | 2024-08-09 02:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T02:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-09 03:05:12.500827+00:00 | 2024-08-09 02:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-QLHERBX1DDR3 | 2024-08-09 03:05:32.136683+00:00 | 2024-08-09 02:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-dsp-reporting", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "DSPExchangeTradeJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T04:00:00+00:00", "--class", "com.viooh.smex.dsp.DSPExchangeTrade", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dsp-exchange-trade/2.11.0/dsp-exchange-trade-2.11.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopyDSPReports", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-dsp-reporting/1.0.0/job.sh", "--environment", "uat", "--reportdate", "2024-08-08", "--region", "cn", "--skipFailure"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-dsp-reporting"}], "Configurations": []} | 2024-08-09 04:00:17.709470+00:00 | 2024-08-08 04:00:00+00:00 | create_emr_steps | exchange-dsp-reporting_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T03:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T03:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T03:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T03:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T03:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T03:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "4", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "3", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "4", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "3", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "4", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "3", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-09T03:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-09T03:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-09 04:00:22.500500+00:00 | 2024-08-09 03:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-3EG50OGXSOBYL | 2024-08-09 04:00:59.553068+00:00 | 2024-08-08 04:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-dsp-reporting_v002 | ||
| return_value | j-2OMS1ITC9Q4XE | 2024-08-09 04:01:04.887122+00:00 | 2024-08-09 03:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T03:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-09 04:05:07.387898+00:00 | 2024-08-09 03:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-BI2CXCQDPEBB | 2024-08-09 04:05:28.837774+00:00 | 2024-08-09 03:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T04:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T04:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T04:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T04:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T04:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T04:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "5", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "4", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "5", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "4", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "5", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "4", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-09T04:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-09T04:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-09 05:00:27.120230+00:00 | 2024-08-09 04:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-16QLARCGH4SBE | 2024-08-09 05:01:10.978996+00:00 | 2024-08-09 04:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T04:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-09 05:05:07.679109+00:00 | 2024-08-09 04:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-3CIIYQQZTP00W | 2024-08-09 05:05:29.626663+00:00 | 2024-08-09 04:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T05:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T05:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T05:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T05:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T05:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T05:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "6", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "5", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "6", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "5", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "6", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "5", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-09T05:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-09T05:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-09 06:00:22.298187+00:00 | 2024-08-09 05:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-AJSXM6VB6E9Z | 2024-08-09 06:00:59.671284+00:00 | 2024-08-09 05:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T05:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-09 06:05:27.818721+00:00 | 2024-08-09 05:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1PS0HMJCZC3GM | 2024-08-09 06:05:45.610022+00:00 | 2024-08-09 05:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T06:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T06:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T06:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T06:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T06:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T06:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "7", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "6", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "7", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "6", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "7", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "6", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-09T06:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-09T06:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-09 07:00:12.756215+00:00 | 2024-08-09 06:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-2WKR5WIZXMPS0 | 2024-08-09 07:00:35.745913+00:00 | 2024-08-09 06:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T06:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-09 07:05:05.300977+00:00 | 2024-08-09 06:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1FYXVUNT39AS9 | 2024-08-09 07:05:27.154133+00:00 | 2024-08-09 06:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T07:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T07:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T07:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T07:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T07:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T07:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "8", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "7", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "8", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "7", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "8", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "7", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-09T07:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-09T07:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-09 08:00:20.671415+00:00 | 2024-08-09 07:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-1GJHS1R1JBEA3 | 2024-08-09 08:01:02.881709+00:00 | 2024-08-09 07:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T07:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-09 08:05:10.235094+00:00 | 2024-08-09 07:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-81A39KIYCXVL | 2024-08-09 08:05:30.123368+00:00 | 2024-08-09 07:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-daily-revenue-report", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-6.14.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 2, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "DailyRevenueReportJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-08T09:00:00+00:00", "--class", "com.viooh.smex.dailyrevenue.DailyRevenueReport", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/daily-revenue-report/1.3.0/daily-revenue-report-1.3.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-daily-revenue-report"}], "Configurations": []} | 2024-08-09 09:00:20.516097+00:00 | 2024-08-08 09:00:00+00:00 | create_emr_steps | exchange-daily-revenue-report_v002 | ||
| return_value | j-1B8BBTKF8MIB9 | 2024-08-09 09:01:07.206493+00:00 | 2024-08-08 09:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-daily-revenue-report_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T08:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-09 09:05:13.477372+00:00 | 2024-08-09 08:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-9IOR4AXVUBPL | 2024-08-09 09:05:31.594112+00:00 | 2024-08-09 08:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T09:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T09:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T09:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T09:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T09:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T09:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "10", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "9", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "10", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "9", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "10", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "9", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-09T09:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-09T09:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-09 10:00:26.220542+00:00 | 2024-08-09 09:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T08:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T08:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T08:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T08:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T08:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T08:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "9", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "8", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "9", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "8", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "9", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "8", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-09T08:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-09T08:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-09 10:03:49.442911+00:00 | 2024-08-09 08:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-3H8ZA2RDK8D5Q | 2024-08-09 10:04:07.836727+00:00 | 2024-08-09 08:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T09:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-09 10:05:20.304925+00:00 | 2024-08-09 09:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1FUSPOTQ623FC | 2024-08-09 10:05:39.523281+00:00 | 2024-08-09 09:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | j-3FLXK4K8XYNHM | 2024-08-09 10:27:53.514764+00:00 | 2024-08-09 09:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T10:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-09 11:05:07.453168+00:00 | 2024-08-09 10:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-3LQK2QYU6Q0W2 | 2024-08-09 11:05:26.551577+00:00 | 2024-08-09 10:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T11:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T11:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T11:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T11:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T11:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T11:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "12", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "11", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "12", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "11", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "12", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "11", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-09T11:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-09T11:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-09 12:00:21.101816+00:00 | 2024-08-09 11:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T10:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T10:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T10:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T10:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T10:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T10:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "11", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "10", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "11", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "10", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "11", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "10", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-09T10:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-09T10:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-09 12:02:32.192014+00:00 | 2024-08-09 10:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-4G5ZYWCMOXC5 | 2024-08-09 12:02:55.023719+00:00 | 2024-08-09 10:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T11:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-09 12:05:17.989587+00:00 | 2024-08-09 11:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1L9AGGBTDNCOI | 2024-08-09 12:05:38.850588+00:00 | 2024-08-09 11:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | j-8HN3DRX4TF4X | 2024-08-09 12:26:35.450048+00:00 | 2024-08-09 11:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T12:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T12:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T12:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T12:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T12:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T12:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "13", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "12", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "13", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "12", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "13", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "12", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-09T12:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-09T12:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-09 13:00:06.688592+00:00 | 2024-08-09 12:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-1MV7GZ4OFYEYS | 2024-08-09 13:00:27.297434+00:00 | 2024-08-09 12:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T12:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-09 13:05:09.928941+00:00 | 2024-08-09 12:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-2XMSH6DV9J65A | 2024-08-09 13:05:29.487134+00:00 | 2024-08-09 12:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T13:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T13:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T13:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T13:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T13:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T13:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "14", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "13", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "14", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "13", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "14", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "13", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-09T13:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-09T13:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-09 14:00:06.869222+00:00 | 2024-08-09 13:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-8AN9L4HCZYHF | 2024-08-09 14:00:25.490128+00:00 | 2024-08-09 13:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T13:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-09 14:05:19.498485+00:00 | 2024-08-09 13:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-1WEB60492HB5E | 2024-08-09 14:05:39.635432+00:00 | 2024-08-09 13:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-0e214a494e67e2266", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T14:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T14:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T14:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T14:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T14:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T14:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "15", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "14", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "15", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "14", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "15", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "14", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-09T14:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-09T14:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-09 15:00:10.984031+00:00 | 2024-08-09 14:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-SRSV8DNR4HO9 | 2024-08-09 15:00:35.638260+00:00 | 2024-08-09 14:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T14:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-09 15:05:10.169207+00:00 | 2024-08-09 14:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-CBV2O4C7WZR8 | 2024-08-09 15:05:30.953517+00:00 | 2024-08-09 14:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 | ||
| return_value | {"Name": "vd-uat-viooh-exchange-trade-summary_v1", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 5, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}, {"InstanceType": "m5.4xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-09a80b42d0e6dedd8", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "SupplyDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T15:00:00+00:00", "--class", "com.viooh.smex.supply.denormalization.SupplyDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/supply-denormalization/2.15.0/supply-denormalization-2.15.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DemandDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T15:00:00+00:00", "--class", "com.viooh.smex.demand.denormalization.DemandDataDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/demand-denormalization/2.14.0/demand-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealSyncDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T15:00:00+00:00", "--class", "com.viooh.smex.dealsync.denormalization.DealSyncDenormalization", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/dealsync-denormalization/2.14.0/dealsync-denormalization-2.14.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ModerationDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T15:00:00+00:00", "--class", "com.viooh.smex.moderation.denormalization.ModerationDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/moderation-denormalization/2.4.0/moderation-denormalization-2.4.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "DealDenormalizationJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T15:00:00+00:00", "--class", "com.viooh.smex.deal.denormalization.DealDenormalization", "--jars", "/usr/lib/spark/external/lib/spark-avro.jar", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/deal-denormalization/1.5.0/deal-denormalization-1.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "ExchangeTradeSummaryJob", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=6g", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", "--conf", "spark.task.maxFailures=20", "--conf", "spark.yarn.maxAppAttempts=10", "--conf", "spark.stage.maxConsecutiveAttempts=20", "--conf", "spark.app.config.key=exchange-trade-summary", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T15:00:00+00:00", "--class", "com.viooh.smex.trade.summary.ExchangeTradeSummary", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/smex/exchange-trade-summary/3.5.0/exchange-trade-summary-3.5.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}, {"Name": "CopySmartExchangeMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "16", "mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "15", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "16", "frame-level-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteFrameLevelMediaOwnerReport", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "15", "frame-level-exchange-lite-mediaowner-trade-report", "frame-level-media-owner-reports", "v1", "csv", "cn-northwest-1"]}}, {"Name": "CopySmartExchangeMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "16", "mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "CopyExchangeLiteMediaOwnerReportV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/1.3.0/job.sh", "uat", "2024-08-09", "15", "exchange-lite-mediaowner-trade-report", "trade-summary-reports", "v2", "json", "cn-northwest-1"]}}, {"Name": "ExportVioohHourLevelReportToDbV2", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-ui-report-v2", "1", "2024-08-09T15:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "deal", "cn-northwest-1"]}}, {"Name": "ExportVioohHourlyCreativeLevelReportToDb", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "s3://cn-northwest-1.elasticmapreduce/libs/script-runner/script-runner.jar", "Args": ["s3://viooh-spark-artifacts-lab-cn/scripts/exchange-trade-summary/sspui_reporting/viooh_sspui_hourly_aggr_report_v2_installer.sh", "uat", "0.0.2", "db-import-viooh-hourly-creative-report", "1", "2024-08-09T15:00:00+00:00", "viooh-spark-artifacts-lab-cn", "scripts/exchange-trade-summary/sspui_reporting", "creative", "cn-northwest-1"]}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "exchange-reporting"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-exchange-trade-summary_v1"}], "Configurations": [{"Classification": "yarn-site", "Properties": {"yarn.resourcemanager.am.max-attempts": "2"}}]} | 2024-08-09 16:00:21.524059+00:00 | 2024-08-09 15:00:00+00:00 | create_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | j-2UANSVS5BJQ5Y | 2024-08-09 16:00:57.653832+00:00 | 2024-08-09 15:00:00+00:00 | create_cluster_and_add_emr_steps | exchange-trade-summary_v1_v002 | ||
| return_value | {"Name": "vd-uat-viooh-vc-asset-import", "LogUri": "s3n://viooh-emr-logs-lab-cn/bigdata/emr/", "ReleaseLabel": "emr-5.35.0", "Instances": {"InstanceFleets": [{"Name": "Masterfleet", "InstanceFleetType": "MASTER", "TargetOnDemandCapacity": 1, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}, {"Name": "Corefleet", "InstanceFleetType": "CORE", "TargetOnDemandCapacity": 3, "InstanceTypeConfigs": [{"InstanceType": "m5.xlarge"}, {"InstanceType": "m5.2xlarge"}]}], "Ec2KeyName": "data.engineer.lab", "KeepJobFlowAliveWhenNoSteps": false, "TerminationProtected": false, "Ec2SubnetId": "subnet-06c5b5a18c64a0316", "EmrManagedMasterSecurityGroup": "sg-0da002beeed87fb1a", "EmrManagedSlaveSecurityGroup": "sg-0f1a0b7517309fcf0", "ServiceAccessSecurityGroup": "sg-009528c399781cc69"}, "BootstrapActions": [], "Applications": [{"Name": "Spark"}, {"Name": "Hadoop"}, {"Name": "Ganglia"}, {"Name": "Zeppelin"}], "VisibleToAllUsers": true, "Steps": [{"Name": "Setup Hadoop Debugging", "ActionOnFailure": "TERMINATE_JOB_FLOW", "HadoopJarStep": {"Jar": "command-runner.jar", "Args": ["state-pusher-script"]}}, {"Name": "PopulateVCAssetsToCassandra", "ActionOnFailure": "TERMINATE_CLUSTER", "HadoopJarStep": {"Args": ["spark-submit", "--deploy-mode", "cluster", "--master", "yarn", "--conf", "spark.yarn.submit.waitAppCompletion=true", "--conf", "spark.driver.memory=9g", "--conf", "spark.app.viooh.compliance.assets.write.flag=true", "--conf", "spark.app.save.latest.frame.state.flag=true", "--conf", "spark.app.viooh.central.raw.assets.write.flag=true", "--conf", "spark.app.env=uat", "--conf", "spark.dag.execution.time=2024-08-09T15:05:00+00:00", "--class", "VioohCentralAssetsMain", "--files", "s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.conf,s3://viooh-spark-artifacts-lab-cn/metrics/batch-job-metrics/1.0.0/job.yaml", "--conf", "spark.metrics.conf=job.conf", "s3://viooh-spark-artifacts-lab-cn/releases/com/viooh/daily-import/1.13.0/daily-import-1.13.0-jar-with-dependencies.jar"], "Jar": "command-runner.jar"}}], "JobFlowRole": "DeEmrInstanceRole", "ServiceRole": "DeEmrServiceRole", "SecurityConfiguration": "bigdata_emr_cn_sec_conf", "Tags": [{"Key": "environment", "Value": "uat"}, {"Key": "role", "Value": "analysis"}, {"Key": "application", "Value": "emr"}, {"Key": "project", "Value": "campaign-compliance"}, {"Key": "TerminationPolicy", "Value": "OFFICE-HOURS"}, {"Key": "ContactEmail", "Value": "viooh.data@viooh.com"}, {"Key": "team", "Value": "bigdata"}, {"Key": "Name", "Value": "vd-uat-viooh-vc-asset-import"}], "Configurations": []} | 2024-08-09 16:05:17.806127+00:00 | 2024-08-09 15:05:00+00:00 | create_emr_steps | vc-asset-import_v002 | ||
| return_value | j-35DK2K6RFGUBC | 2024-08-09 16:11:17.398258+00:00 | 2024-08-09 15:05:00+00:00 | create_cluster_and_add_emr_steps | vc-asset-import_v002 |