Attribute |
Value |
dag_id |
primary-audience-global-ind_v003 |
duration |
3.130161 |
end_date |
2021-04-13 09:54:48.898563+00:00 |
execution_date |
2021-04-13T09:54:00.917790+00:00 |
executor_config |
{} |
generate_command |
<function TaskInstance.generate_command at 0x7f5fe9ba2730> |
hostname |
airflow-worker-0.airflow-worker.bigdata.svc.cluster.local |
is_premature |
False |
job_id |
16684 |
key |
('primary-audience-global-ind_v003', 'create_cluster', <Pendulum [2021-04-13T09:54:00.917790+00:00]>, 2) |
log |
<Logger airflow.task (INFO)> |
log_filepath |
/opt/airflow/logs/primary-audience-global-ind_v003/create_cluster/2021-04-13T09:54:00.917790+00:00.log |
log_url |
https://airflow.devel.viooh.net.cn/admin/airflow/log?execution_date=2021-04-13T09%3A54%3A00.917790%2B00%3A00&task_id=create_cluster&dag_id=primary-audience-global-ind_v003 |
logger |
<Logger airflow.task (INFO)> |
mark_success_url |
https://airflow.devel.viooh.net.cn/success?task_id=create_cluster&dag_id=primary-audience-global-ind_v003&execution_date=2021-04-13T09%3A54%3A00.917790%2B00%3A00&upstream=false&downstream=false |
max_tries |
0 |
metadata |
MetaData(bind=None) |
next_try_number |
2 |
operator |
PythonOperator |
pid |
132889 |
pool |
default_pool |
pool_slots |
1 |
prev_attempted_tries |
1 |
previous_execution_date_success |
2021-04-13 06:16:58.960524+00:00 |
previous_start_date_success |
2021-04-13 06:17:27.867948+00:00 |
previous_ti |
<TaskInstance: primary-audience-global-ind_v003.create_cluster 2021-04-13 06:16:58.960524+00:00 [success]> |
previous_ti_success |
<TaskInstance: primary-audience-global-ind_v003.create_cluster 2021-04-13 06:16:58.960524+00:00 [success]> |
priority_weight |
1 |
queue |
default |
queued_dttm |
2021-04-13 09:54:43.389519+00:00 |
raw |
False |
run_as_user |
None |
start_date |
2021-04-13 09:54:45.768402+00:00 |
state |
success |
task |
<Task(PythonOperator): create_cluster> |
task_id |
create_cluster |
test_mode |
False |
try_number |
2 |
unixname |
airflow |
Attribute |
Value |
dag |
<DAG: primary-audience-global-ind_v003> |
dag_id |
primary-audience-global-ind_v003 |
depends_on_past |
False |
deps |
{<TIDep(Not Previously Skipped)>, <TIDep(Not In Retry Period)>, <TIDep(Trigger Rule)>, <TIDep(Previous Dagrun State)>} |
do_xcom_push |
True |
downstream_list |
[] |
downstream_task_ids |
set() |
email |
None |
email_on_failure |
True |
email_on_retry |
True |
end_date |
None |
execution_timeout |
None |
executor_config |
{} |
extra_links |
[] |
global_operator_extra_link_dict |
{} |
inlets |
[] |
lineage_data |
None |
log |
<Logger airflow.task.operators (INFO)> |
logger |
<Logger airflow.task.operators (INFO)> |
max_retry_delay |
None |
on_failure_callback |
<function task_fail_slack_alert at 0x7f5fe02ed9d8> |
on_retry_callback |
None |
on_success_callback |
None |
op_args |
[] |
op_kwargs |
{'cluster-name': 'primary-audience-global-JCDECAUX_CN_SH_METRO-2408', 'audience-config-file': 'primary_audience_global_ind_conf', 'master-instance-types': 'm5.2xlarge,m5.4xlarge', 'core-instance-types': 'm5.2xlarge,m5.4xlarge', 'task-instance-types': 'm5.2xlarge,m5.4xlarge', 'core-instance-capacity': 1, 'task-instance-capacity': 0, 'emr-version': 'emr-6.0.0', 'audience-load-bootstrap': True, 'emr-steps': '[{\n "step-name": "PrimaryAudienceGlobal-integrity",\n "py-script": "jobs/integrity_job.py",\n "config-json": [\n {"spark.app.audience.country":"CN"},\n {"spark.app.audience.media.owner":"JCDECAUX_CN_SH_METRO"},\n {"spark.app.audience.version":"2408"},\n {"spark.app.audience.bucket.name":"s3://jcd-prd-datacorp-partners"},\n {"spark.app.audience.digital.file":"viooh/outputs/china/metro/shanghai/audience/shanghai-20240910030713/data/type=dynamic/"},\n {"spark.app.audience.static.file":"viooh/outputs/china/metro/shanghai/audience/shanghai-20240910030713/data/type=static/"},\n {"spark.app.audience.demographic.file":"viooh/outputs/china/metro/shanghai/audience/shanghai-20240910030713/demographics/"},\n {"spark.app.audience.input.bucket":"s3://global-cn-audiences-input"},\n {"spark.app.audience.output.bucket":"s3://global-cn-audiences-output"},\n {"spark.app.audience.folder":"route"},\n {"spark.app.audience.computation.date":"20240917141929"},\n {"spark.executor.memoryOverhead":"2500"}\n ],\n "artifact": "AudiencesPipeline"\n }]', 'dag-id': 'primary-audience-global-ind_v003', 'schedule_interval': None, 'trigger_dags': []} |
operator_extra_link_dict |
{} |
operator_extra_links |
() |
outlets |
[] |
owner |
data.engineers@viooh.com |
params |
{} |
pool |
default_pool |
pool_slots |
1 |
priority_weight |
1 |
priority_weight_total |
1 |
provide_context |
True |
queue |
default |
resources |
None |
retries |
0 |
retry_delay |
0:05:00 |
retry_exponential_backoff |
False |
run_as_user |
None |
schedule_interval |
None |
shallow_copy_attrs |
('python_callable', 'op_kwargs') |
sla |
None |
start_date |
2024-09-16 00:00:00+00:00 |
subdag |
None |
task_concurrency |
None |
task_id |
create_cluster |
task_type |
PythonOperator |
template_ext |
[] |
template_fields |
('templates_dict', 'op_args', 'op_kwargs') |
templates_dict |
None |
trigger_rule |
all_success |
ui_color |
#ffefeb |
ui_fgcolor |
#000 |
upstream_list |
[<Task(PythonOperator): validation>] |
upstream_task_ids |
{'validation'} |
wait_for_downstream |
False |
weight_rule |
downstream |