From 3c8ded845c446f6829c126bdd39e7560e4d5dafb Mon Sep 17 00:00:00 2001 From: Sahand Date: Wed, 27 Aug 2025 09:00:03 +0330 Subject: [PATCH] Cache: fixed cache stuck UPDATING if worker crashes If use_stale and cache_lock used together, if the worker doing the update crashes, the flag never resets from updating and we will get stuck in UPDATING state forever. --- src/event/ngx_event.c | 8 ++++++-- src/event/ngx_event.h | 2 +- src/http/ngx_http_cache.h | 2 ++ src/http/ngx_http_file_cache.c | 14 ++++++++++++++ src/os/unix/ngx_process.c | 2 ++ 5 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/event/ngx_event.c b/src/event/ngx_event.c index ef525d93b..860dbfb35 100644 --- a/src/event/ngx_event.c +++ b/src/event/ngx_event.c @@ -76,7 +76,7 @@ static ngx_atomic_t ngx_stat_waiting0; ngx_atomic_t *ngx_stat_waiting = &ngx_stat_waiting0; #endif - +ngx_pid_t *ngx_active_workers_pids; static ngx_command_t ngx_events_commands[] = { @@ -566,6 +566,7 @@ ngx_event_module_init(ngx_cycle_t *cycle) + cl; /* ngx_stat_waiting */ #endif + size += ngx_min(NGX_MAX_PROCESSES * sizeof(ngx_pid_t), cl); shm.size = size; ngx_str_set(&shm.name, "nginx_shared_zone"); @@ -610,7 +611,10 @@ ngx_event_module_init(ngx_cycle_t *cycle) ngx_stat_reading = (ngx_atomic_t *) (shared + 7 * cl); ngx_stat_writing = (ngx_atomic_t *) (shared + 8 * cl); ngx_stat_waiting = (ngx_atomic_t *) (shared + 9 * cl); - + + ngx_active_workers_pids = (ngx_pid_t *) (shared + 11 * cl); +#else + ngx_active_workers_pids = (ngx_pid_t *) (shared + 3 * cl); #endif return NGX_OK; diff --git a/src/event/ngx_event.h b/src/event/ngx_event.h index deac04e7b..69a4b42f8 100644 --- a/src/event/ngx_event.h +++ b/src/event/ngx_event.h @@ -476,7 +476,7 @@ extern ngx_atomic_t *ngx_stat_writing; extern ngx_atomic_t *ngx_stat_waiting; #endif - +extern ngx_pid_t *ngx_active_workers_pids; #define NGX_UPDATE_TIME 1 #define NGX_POST_EVENTS 2 diff --git a/src/http/ngx_http_cache.h b/src/http/ngx_http_cache.h index bb936c5fe..8df0edf60 100644 --- a/src/http/ngx_http_cache.h +++ b/src/http/ngx_http_cache.h @@ -59,6 +59,8 @@ typedef struct { size_t body_start; off_t fs_size; ngx_msec_t lock_time; + ngx_int_t updating_updater_process_slot; + ngx_pid_t updating_updater_process_pid; } ngx_http_file_cache_node_t; diff --git a/src/http/ngx_http_file_cache.c b/src/http/ngx_http_file_cache.c index 5209f003b..0327cc585 100644 --- a/src/http/ngx_http_file_cache.c +++ b/src/http/ngx_http_file_cache.c @@ -657,10 +657,24 @@ ngx_http_file_cache_read(ngx_http_request_t *r, ngx_http_cache_t *c) ngx_shmtx_lock(&cache->shpool->mutex); + if (c->node->updating && + ngx_active_workers_pids[c->node->updating_updater_process_slot] + != c->node->updating_updater_process_pid) { + /* + if the active pid in the index is different from + the initiator, it means the original worker has + crashed. reset the updating flag so another worker + can handle the cache update. + */ + c->node->updating = 0; + } + if (c->node->updating) { rc = NGX_HTTP_CACHE_UPDATING; } else { + c->node->updating_updater_process_slot = ngx_process_slot; + c->node->updating_updater_process_pid = ngx_pid; c->node->updating = 1; c->updating = 1; c->lock_time = c->node->lock_time; diff --git a/src/os/unix/ngx_process.c b/src/os/unix/ngx_process.c index 15680237a..38f721397 100644 --- a/src/os/unix/ngx_process.c +++ b/src/os/unix/ngx_process.c @@ -205,6 +205,8 @@ ngx_spawn_process(ngx_cycle_t *cycle, ngx_spawn_proc_pt proc, void *data, ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "start %s %P", name, pid); + ngx_active_workers_pids[s] = pid; + ngx_processes[s].pid = pid; ngx_processes[s].exited = 0;