diff options
Diffstat (limited to 'src/http/ngx_http_file_cache.c')
| -rw-r--r-- | src/http/ngx_http_file_cache.c | 1308 |
1 files changed, 1166 insertions, 142 deletions
diff --git a/src/http/ngx_http_file_cache.c b/src/http/ngx_http_file_cache.c index 9f93bd8f5..0c2b9e2d9 100644 --- a/src/http/ngx_http_file_cache.c +++ b/src/http/ngx_http_file_cache.c @@ -7,253 +7,1277 @@ #include <ngx_config.h> #include <ngx_core.h> #include <ngx_http.h> +#include <ngx_md5.h> -#if (NGX_HAVE_OPENSSL_MD5_H) -#include <openssl/md5.h> -#else -#include <md5.h> -#endif +static ngx_int_t ngx_http_file_cache_exists(ngx_http_request_t *r, + ngx_http_file_cache_t *cache); +static ngx_http_file_cache_node_t * + ngx_http_file_cache_lookup(ngx_http_file_cache_t *cache, u_char *key); +static void ngx_http_file_cache_rbtree_insert_value(ngx_rbtree_node_t *temp, + ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel); +static void ngx_http_file_cache_cleanup(void *data); +static time_t ngx_http_file_cache_expire(ngx_http_file_cache_t *cache, + ngx_uint_t force); +static ngx_int_t ngx_http_file_cache_clean_noop(ngx_tree_ctx_t *ctx, + ngx_str_t *path); +static ngx_int_t ngx_http_file_cache_clean_file(ngx_tree_ctx_t *ctx, + ngx_str_t *path); -#if (NGX_OPENSSL_MD5) -#define MD5Init MD5_Init -#define MD5Update MD5_Update -#define MD5Final MD5_Final -#endif +static u_char ngx_http_file_cache_key[] = { LF, 'K', 'E', 'Y', ':', ' ' }; -ngx_int_t ngx_http_file_cache_get(ngx_http_request_t *r, - ngx_http_cache_ctx_t *ctx) + +static ngx_int_t +ngx_http_file_cache_init(ngx_shm_zone_t *shm_zone, void *data) { - ngx_uint_t i; + ngx_http_file_cache_t *ocache = data; + + ngx_rbtree_node_t *sentinel; + ngx_http_file_cache_t *cache; + + cache = shm_zone->data; + + if (ocache) { + if (ngx_strcmp(cache->path->name.data, ocache->path->name.data) != 0) { + ngx_log_error(NGX_LOG_EMERG, shm_zone->shm.log, 0, + "cache \"%V\" uses the \"%V\" cache path " + "while previously it used the \"%V\" cache path", + &shm_zone->name, &cache->path->name, + &ocache->path->name); + + return NGX_ERROR; + } + + cache->rbtree = ocache->rbtree; + cache->queue = ocache->queue; + cache->shpool = ocache->shpool; + cache->created = ocache->created; + + return NGX_OK; + } + + cache->shpool = (ngx_slab_pool_t *) shm_zone->shm.addr; + + cache->rbtree = ngx_slab_alloc(cache->shpool, sizeof(ngx_rbtree_t)); + if (cache->rbtree == NULL) { + return NGX_ERROR; + } + + sentinel = ngx_slab_alloc(cache->shpool, sizeof(ngx_rbtree_node_t)); + if (sentinel == NULL) { + return NGX_ERROR; + } + + ngx_rbtree_init(cache->rbtree, sentinel, + ngx_http_file_cache_rbtree_insert_value); + + cache->queue = ngx_slab_alloc(cache->shpool, sizeof(ngx_queue_t)); + if (cache->queue == NULL) { + return NGX_ERROR; + } + + ngx_queue_init(cache->queue); + + cache->created = ngx_time(); + + return NGX_OK; +} + + +void +ngx_http_file_cache_create_key(ngx_http_request_t *r) +{ + size_t len; ngx_str_t *key; + ngx_uint_t i; + ngx_md5_t md5; ngx_http_cache_t *c; - MD5_CTX md5; c = r->cache; - c->file.name.len = ctx->path->name.len + 1 + ctx->path->len + 32; - if (!(c->file.name.data = ngx_palloc(r->pool, c->file.name.len + 1))) { - return NGX_ABORT; + len = 0; + + ngx_crc32_init(c->crc32); + ngx_md5_init(&md5); + + key = c->keys.elts; + for (i = 0; i < c->keys.nelts; i++) { + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http cache key: \"%V\"", &key[i]); + + len += key[i].len; + + ngx_crc32_update(&c->crc32, key[i].data, key[i].len); + ngx_md5_update(&md5, key[i].data, key[i].len); } - MD5Init(&md5); + c->header_start = sizeof(ngx_http_file_cache_header_t) + + sizeof(ngx_http_file_cache_key) + len + 1; - key = c->key.elts; - for (i = 0; i < c->key.nelts; i++) { - MD5Update(&md5, key[i].data, key[i].len); + ngx_crc32_final(c->crc32); + ngx_md5_final(c->key, &md5); +} + + +ngx_int_t +ngx_http_file_cache_open(ngx_http_request_t *r) +{ + u_char *p; + time_t now; + ssize_t n; + ngx_int_t rc, rv; + ngx_uint_t cold, test; + ngx_path_t *path; + ngx_http_cache_t *c; + ngx_pool_cleanup_t *cln; + ngx_open_file_info_t of; + ngx_http_file_cache_t *cache; + ngx_http_core_loc_conf_t *clcf; + ngx_http_file_cache_header_t *h; + + c = r->cache; + cache = c->file_cache; + + cln = ngx_pool_cleanup_add(r->pool, 0); + if (cln == NULL) { + return NGX_ERROR; } - MD5Update(&md5, ctx->key.data, ctx->key.len); + rc = ngx_http_file_cache_exists(r, cache); - MD5Final(c->md5, &md5); + ngx_log_debug3(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http file cache exists: %i u:%ui e:%d", + rc, c->uses, c->exists); - ngx_memcpy(c->file.name.data, ctx->path->name.data, ctx->path->name.len); + if (rc == NGX_ERROR) { + return rc; + } - ngx_md5_text(c->file.name.data + ctx->path->name.len + 1 + ctx->path->len, - c->md5); + cln->handler = ngx_http_file_cache_cleanup; + cln->data = c; - ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, - "file cache key: %V, md5: %s", &ctx->key, - c->file.name.data + ctx->path->name.len + 1 + ctx->path->len); + if (rc == NGX_AGAIN) { + return rc; + } - ngx_create_hashed_filename(&c->file, ctx->path); + now = ngx_time(); - ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, - "file cache name: %s", c->file.name.data); + cold = (now - cache->created < cache->inactive) ? 1 : 0; - return ngx_http_file_cache_open(r->cache); -} + if (rc == NGX_OK) { + if (c->error) { + return c->error; + } -ngx_int_t ngx_http_file_cache_open(ngx_http_cache_t *c) -{ - ssize_t n; - ngx_err_t err; - ngx_http_cache_header_t *h; + c->temp_file = 1; + test = c->exists ? 1 : 0; + rv = NGX_DECLINED; - c->file.fd = ngx_open_file(c->file.name.data, - NGX_FILE_RDONLY, NGX_FILE_OPEN); + } else { /* rc == NGX_DECLINED */ - if (c->file.fd == NGX_INVALID_FILE) { - err = ngx_errno; + if (c->min_uses > 1) { - if (err == NGX_ENOENT || err == NGX_ENOTDIR) { - return NGX_DECLINED; + if (!cold) { + return NGX_AGAIN; + } + + test = 1; + rv = NGX_AGAIN; + + } else { + c->temp_file = 1; + test = cold ? 1 : 0; + rv = NGX_DECLINED; } + } - ngx_log_error(NGX_LOG_CRIT, c->log, ngx_errno, - ngx_open_file_n " \"%s\" failed", c->file.name.data); + path = cache->path; + + c->file.name.len = path->name.len + 1 + path->len + + 2 * NGX_HTTP_CACHE_KEY_LEN; + + c->file.name.data = ngx_pnalloc(r->pool, c->file.name.len + 1); + if (c->file.name.data == NULL) { return NGX_ERROR; } - if (c->uniq) { - if (ngx_fd_info(c->file.fd, &c->file.info) == NGX_FILE_ERROR) { - ngx_log_error(NGX_LOG_CRIT, c->log, ngx_errno, - ngx_fd_info_n " \"%s\" failed", c->file.name.data); + ngx_memcpy(c->file.name.data, path->name.data, path->name.len); + + p = c->file.name.data + path->name.len + 1 + path->len; + p = ngx_hex_dump(p, c->key, NGX_HTTP_CACHE_KEY_LEN); + *p = '\0'; + ngx_create_hashed_filename(path, c->file.name.data, c->file.name.len); + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, + "cache file: \"%s\"", c->file.name.data); + + if (!test) { + return NGX_DECLINED; + } + + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); + + ngx_memzero(&of, sizeof(ngx_open_file_info_t)); + + of.uniq = c->uniq; + of.valid = clcf->open_file_cache_valid; + of.min_uses = clcf->open_file_cache_min_uses; + of.events = clcf->open_file_cache_events; + of.directio = NGX_OPEN_FILE_DIRECTIO_OFF; + + if (ngx_open_cached_file(clcf->open_file_cache, &c->file.name, &of, r->pool) + != NGX_OK) + { + switch (of.err) { + + case 0: return NGX_ERROR; - } - if (ngx_file_uniq(&c->file.info) == c->uniq) { - if (ngx_close_file(c->file.fd) == NGX_FILE_ERROR) { - ngx_log_error(NGX_LOG_ALERT, c->log, ngx_errno, - ngx_close_file_n " \"%s\" failed", - c->file.name.data); - } + case NGX_ENOENT: + case NGX_ENOTDIR: + return rv; - return NGX_HTTP_CACHE_THE_SAME; + default: + ngx_log_error(NGX_LOG_CRIT, r->connection->log, of.err, + ngx_open_file_n " \"%s\" failed", c->file.name.data); + return NGX_ERROR; } } - n = ngx_read_file(&c->file, c->buf->pos, c->buf->end - c->buf->last, 0); + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http file cache fd: %d", of.fd); + + c->file.fd = of.fd; - if (n == NGX_ERROR || n == NGX_AGAIN) { + c->buf = ngx_create_temp_buf(r->pool, c->body_start); + if (c->buf == NULL) { + return NGX_ERROR; + } + + n = ngx_read_file(&c->file, c->buf->pos, c->body_start, 0); + + if (n == NGX_ERROR) { return n; } - if (n <= c->header_size) { - ngx_log_error(NGX_LOG_CRIT, c->log, 0, + if ((size_t) n <= c->header_start) { + ngx_log_error(NGX_LOG_CRIT, r->connection->log, 0, "cache file \"%s\" is too small", c->file.name.data); return NGX_ERROR; } - h = (ngx_http_cache_header_t *) c->buf->pos; - c->expires = h->expires; - c->last_modified= h->last_modified; - c->date = h->date; - c->length = h->length; + h = (ngx_http_file_cache_header_t *) c->buf->pos; - if (h->key_len > (size_t) (c->buf->end - c->buf->pos)) { - ngx_log_error(NGX_LOG_ALERT, c->log, 0, - "cache file \"%s\" is probably invalid", - c->file.name.data); + if (h->crc32 != c->crc32 || (size_t) h->header_start != c->header_start) { + ngx_log_error(NGX_LOG_CRIT, r->connection->log, 0, + "cache file \"%s\" has md5 collision", c->file.name.data); return NGX_DECLINED; } -#if 0 + c->buf->last += n; - /* TODO */ + c->valid_sec = h->valid_sec; + c->last_modified = h->last_modified; + c->date = h->date; + c->valid_msec = h->valid_msec; + c->length = of.size; + c->body_start = h->body_start; - if (c->key_len && h->key_len != c->key_len) { + r->cached = 1; - ngx_strncmp(h->key, c->key_data, h->key_len) != 0)) + if (cold) { - h->key[h->key_len] = '\0'; - ngx_log_error(NGX_LOG_ALERT, c->log, 0, - "md5 collision: \"%s\" and \"%s\"", - h->key, c->key.data); - return NGX_DECLINED; + ngx_shmtx_lock(&cache->shpool->mutex); + + c->node->uses = c->min_uses; + c->node->body_start = c->body_start; + c->node->exists = 1; + + ngx_shmtx_unlock(&cache->shpool->mutex); } -#endif + if (c->valid_sec < now) { - c->buf->last += n; + c->uses = c->min_uses; + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http file cache expired: %T %T", c->valid_sec, now); - if (c->expires < ngx_time()) { - ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, - "http file cache expired"); return NGX_HTTP_CACHE_STALE; } /* TODO: NGX_HTTP_CACHE_AGED */ - /* STUB */ return NGX_DECLINED; - return NGX_OK; } -#if 0 +static ngx_int_t +ngx_http_file_cache_exists(ngx_http_request_t *r, ngx_http_file_cache_t *cache) +{ + ngx_int_t rc; + ngx_http_file_cache_node_t *fcn; + + ngx_shmtx_lock(&cache->shpool->mutex); + + fcn = ngx_http_file_cache_lookup(cache, r->cache->key); + + if (fcn) { + ngx_queue_remove(&fcn->queue); + + if (fcn->error) { + + if (fcn->valid_sec < ngx_time()) { + goto renew; + } + + rc = NGX_OK; + + goto done; + } + + fcn->uses++; + fcn->count++; + + if (fcn->exists) { + + r->cache->exists = fcn->exists; + r->cache->body_start = fcn->body_start; + + rc = NGX_OK; + + goto done; + } + + if (fcn->uses >= r->cache->min_uses) { + + r->cache->exists = fcn->exists; + + if (fcn->body_start) { + r->cache->body_start = fcn->body_start; + } + + rc = NGX_OK; + + } else { + rc = NGX_AGAIN; + } + + goto done; + } + + fcn = ngx_slab_alloc_locked(cache->shpool, + sizeof(ngx_http_file_cache_node_t)); + if (fcn == NULL) { + ngx_shmtx_unlock(&cache->shpool->mutex); + + (void) ngx_http_file_cache_expire(cache, 1); + + ngx_shmtx_lock(&cache->shpool->mutex); + + fcn = ngx_slab_alloc_locked(cache->shpool, + sizeof(ngx_http_file_cache_node_t)); + if (fcn == NULL) { + rc = NGX_ERROR; + goto failed; + } + } + + ngx_memcpy((u_char *) &fcn->node.key, r->cache->key, + sizeof(ngx_rbtree_key_t)); + + ngx_memcpy(fcn->key, &r->cache->key[sizeof(ngx_rbtree_key_t)], + NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t)); + + ngx_rbtree_insert(cache->rbtree, &fcn->node); + +renew: + + rc = NGX_DECLINED; + + fcn->uses = 1; + fcn->count = 1; + fcn->valid_msec = 0; + fcn->error = 0; + fcn->exists = 0; + fcn->valid_sec = 0; + fcn->uniq = 0; + fcn->body_start = 0; + +done: + + fcn->expire = ngx_time() + cache->inactive; + + ngx_queue_insert_head(cache->queue, &fcn->queue); + + r->cache->uniq = fcn->uniq; + r->cache->uses = fcn->uses; + r->cache->error = fcn->error; + r->cache->node = fcn; + +failed: + + ngx_shmtx_unlock(&cache->shpool->mutex); + + return rc; +} + + +static ngx_http_file_cache_node_t * +ngx_http_file_cache_lookup(ngx_http_file_cache_t *cache, u_char *key) +{ + ngx_int_t rc; + ngx_rbtree_key_t node_key; + ngx_rbtree_node_t *node, *sentinel; + ngx_http_file_cache_node_t *fcn; + + ngx_memcpy((u_char *) &node_key, key, sizeof(ngx_rbtree_key_t)); + + node = cache->rbtree->root; + sentinel = cache->rbtree->sentinel; + + while (node != sentinel) { + + if (node_key < node->key) { + node = node->left; + continue; + } + + if (node_key > node->key) { + node = node->right; + continue; + } + + /* node_key == node->key */ + + do { + fcn = (ngx_http_file_cache_node_t *) node; + + rc = ngx_memcmp(&key[sizeof(ngx_rbtree_key_t)], fcn->key, + NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t)); + + if (rc == 0) { + return fcn; + } + + node = (rc < 0) ? node->left : node->right; + + } while (node != sentinel && node_key == node->key); + + break; + } + + /* not found */ + + return NULL; +} + + +static void +ngx_http_file_cache_rbtree_insert_value(ngx_rbtree_node_t *temp, + ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel) +{ + ngx_rbtree_node_t **p; + ngx_http_file_cache_node_t *cn, *cnt; + + for ( ;; ) { + + if (node->key < temp->key) { + + p = &temp->left; + + } else if (node->key > temp->key) { + + p = &temp->right; + + } else { /* node->key == temp->key */ + + cn = (ngx_http_file_cache_node_t *) node; + cnt = (ngx_http_file_cache_node_t *) temp; + + p = (ngx_memcmp(cn->key, cnt->key, + NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t)) + < 0) + ? &temp->left : &temp->right; + } + + if (*p == sentinel) { + break; + } + + temp = *p; + } + + *p = node; + node->parent = temp; + node->left = sentinel; + node->right = sentinel; + ngx_rbt_red(node); +} + + +void +ngx_http_file_cache_set_header(ngx_http_request_t *r, u_char *buf) +{ + ngx_http_file_cache_header_t *h = (ngx_http_file_cache_header_t *) buf; + + u_char *p; + ngx_str_t *key; + ngx_uint_t i; + ngx_http_cache_t *c; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http file cache set header"); + + c = r->cache; + + h->valid_sec = c->valid_sec; + h->last_modified = c->last_modified; + h->date = c->date; + h->crc32 = c->crc32; + h->valid_msec = (u_short) c->valid_msec; + h->header_start = (u_short) c->header_start; + h->body_start = (u_short) c->body_start; + + p = buf + sizeof(ngx_http_file_cache_header_t); + + p = ngx_cpymem(p, ngx_http_file_cache_key, sizeof(ngx_http_file_cache_key)); + + key = c->keys.elts; + for (i = 0; i < c->keys.nelts; i++) { + p = ngx_copy(p, key[i].data, key[i].len); + } + + *p = LF; +} + + +void +ngx_http_file_cache_update(ngx_http_request_t *r, ngx_temp_file_t *tf) +{ + ngx_int_t rc; + ngx_file_uniq_t uniq; + ngx_file_info_t fi; + ngx_http_cache_t *c; + ngx_ext_rename_file_t ext; + ngx_http_file_cache_t *cache; + + c = r->cache; + + if (c->updated) { + return; + } + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http file cache update"); + + c->updated = 1; + + cache = c->file_cache; + + uniq = 0; + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http file cache rename: \"%s\" to \"%s\"", + tf->file.name.data, c->file.name.data); + + ext.access = NGX_FILE_OWNER_ACCESS; + ext.path_access = NGX_FILE_OWNER_ACCESS; + ext.time = -1; + ext.create_path = 1; + ext.delete_file = 1; + ext.log_rename_error = 1; + ext.log = r->connection->log; + + rc = ngx_ext_rename_file(&tf->file.name, &c->file.name, &ext); + + if (rc == NGX_OK) { + + if (ngx_fd_info(tf->file.fd, &fi) == NGX_FILE_ERROR) { + ngx_log_error(NGX_LOG_CRIT, r->connection->log, ngx_errno, + ngx_fd_info_n " \"%s\" failed", tf->file.name.data); + + rc = NGX_ERROR; + + } else { + uniq = ngx_file_uniq(&fi); + } + } + + ngx_shmtx_lock(&cache->shpool->mutex); + + c->node->count--; + c->node->uniq = uniq; + c->node->body_start = c->body_start; + + if (rc == NGX_OK) { + c->node->exists = 1; + } + + ngx_shmtx_unlock(&cache->shpool->mutex); +} + + +ngx_int_t +ngx_http_cache_send(ngx_http_request_t *r) +{ + ngx_int_t rc; + ngx_buf_t *b; + ngx_chain_t out; + ngx_http_cache_t *c; + + c = r->cache; + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http file cache send: %s", c->file.name.data); + + /* we need to allocate all before the header would be sent */ + + b = ngx_pcalloc(r->pool, sizeof(ngx_buf_t)); + if (b == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + b->file = ngx_pcalloc(r->pool, sizeof(ngx_file_t)); + if (b->file == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + rc = ngx_http_send_header(r); + + if (rc == NGX_ERROR || rc > NGX_OK || r->header_only) { + return rc; + } + + b->file_pos = c->body_start; + b->file_last = c->length; + + b->in_file = (c->length - c->body_start) ? 1: 0; + b->last_buf = (r == r->main) ? 1: 0; + b->last_in_chain = 1; + + b->file->fd = c->file.fd; + b->file->name = c->file.name; + b->file->log = r->connection->log; + + out.buf = b; + out.next = NULL; + + return ngx_http_output_filter(r, &out); +} + + +void +ngx_http_file_cache_free(ngx_http_request_t *r, ngx_temp_file_t *tf) +{ + ngx_http_cache_t *c; + ngx_http_file_cache_t *cache; + + c = r->cache; + + if (c->updated) { + return; + } + + c->updated = 1; + + cache = c->file_cache; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http file cache free"); + + ngx_shmtx_lock(&cache->shpool->mutex); + + c->node->count--; + + if (c->error) { + c->node->valid_sec = c->valid_sec; + c->node->valid_msec = c->valid_msec; + c->node->error = c->error; + } + + ngx_shmtx_unlock(&cache->shpool->mutex); + + if (c->temp_file) { + if (tf && tf->file.fd != NGX_INVALID_FILE) { + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http file cache incomplete: \"%s\"", + tf->file.name.data); + + if (ngx_delete_file(tf->file.name.data) == NGX_FILE_ERROR) { + ngx_log_error(NGX_LOG_CRIT, r->connection->log, ngx_errno, + ngx_delete_file_n " \"%s\" failed", + tf->file.name.data); + } + } + } +} + + +static void +ngx_http_file_cache_cleanup(void *data) +{ + ngx_http_cache_t *c = data; + + ngx_http_file_cache_t *cache; + + if (c->updated) { + return; + } + + c->updated = 1; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->file.log, 0, + "http file cache cleanup"); + + if (c->error) { + return; + } + + cache = c->file_cache; + + ngx_shmtx_lock(&cache->shpool->mutex); + + c->node->count--; + + ngx_shmtx_unlock(&cache->shpool->mutex); +} -int ngx_http_cache_update_file(ngx_http_request_t *r, ngx_http_cache_ctx_t *ctx, - ngx_str_t *temp_file) +static time_t +ngx_http_file_cache_expire(ngx_http_file_cache_t *cache, ngx_uint_t forced) { - int retry; - ngx_err_t err; + u_char *name, *p; + size_t len; + time_t now, wait; + ngx_uint_t tries; + ngx_path_t *path; + ngx_queue_t *q; + ngx_http_file_cache_node_t *fcn; + u_char key[2 * NGX_HTTP_CACHE_KEY_LEN]; - retry = 0; + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, + "http file cache expire"); + + path = cache->path; + len = path->name.len + 1 + path->len + 2 * NGX_HTTP_CACHE_KEY_LEN; + + now = ngx_time(); + + ngx_shmtx_lock(&cache->shpool->mutex); + + tries = 0; for ( ;; ) { - if (ngx_rename_file(temp_file->data, ctx->file.name.data) == NGX_OK) { - return NGX_OK; + + if (ngx_queue_empty(cache->queue)) { + wait = cache->inactive; + break; } - err = ngx_errno; + q = ngx_queue_last(cache->queue); + + fcn = ngx_queue_data(q, ngx_http_file_cache_node_t, queue); + + if (!forced) { + wait = fcn->expire - now; -#if (NGX_WIN32) - if (err == NGX_EEXIST) { - if (ngx_win32_rename_file(temp_file, &ctx->file.name, r->pool) - == NGX_ERROR) - { - return NGX_ERROR; + if (wait > 0) { + break; } } -#endif - if (retry || (err != NGX_ENOENT && err != NGX_ENOTDIR)) { - ngx_log_error(NGX_LOG_CRIT, r->connection->log, err, - ngx_rename_file_n "(\"%s\", \"%s\") failed", - temp_file->data, ctx->file.name.data); + ngx_log_debug6(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, + "http file cache expire: #%d %d %02xd%02xd%02xd%02xd", + fcn->count, fcn->exists, + fcn->key[0], fcn->key[1], fcn->key[2], fcn->key[3]); - return NGX_ERROR; + if (fcn->count) { + + if (!forced) { + + p = ngx_hex_dump(key, (u_char *) &fcn->node.key, + sizeof(ngx_rbtree_key_t)); + (void) ngx_hex_dump(p, fcn->key, + NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t)); + + /* + * abnormally exited workers may leave locked cache entries, + * and although it may be safe to remove them completely, + * we prefer to remove them from inactive queue and rbtree + * only, and to allow other leaks + */ + + ngx_queue_remove(q); + + ngx_rbtree_delete(cache->rbtree, &fcn->node); + + ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0, + "ignore long locked inactive cache entry %*s", + NGX_HTTP_CACHE_KEY_LEN, key); + } + + if (tries++ < 10) { + continue; + } + + wait = 1; + break; } - if (ngx_create_path(&ctx->file, ctx->path) == NGX_ERROR) { - return NGX_ERROR; + forced = 0; + + if (!fcn->exists) { + + ngx_queue_remove(q); + + ngx_rbtree_delete(cache->rbtree, &fcn->node); + + ngx_slab_free_locked(cache->shpool, fcn); + + continue; + } + + name = ngx_alloc(len + 1, ngx_cycle->log); + if (name == NULL) { + wait = 60; + break; } - retry = 1; + ngx_memcpy(name, path->name.data, path->name.len); + + p = name + path->name.len + 1 + path->len; + p = ngx_hex_dump(p, (u_char *) &fcn->node.key, + sizeof(ngx_rbtree_key_t)); + p = ngx_hex_dump(p, fcn->key, + NGX_HTTP_CACHE_KEY_LEN - sizeof(ngx_rbtree_key_t)); + *p = '\0'; + + ngx_queue_remove(q); + + ngx_rbtree_delete(cache->rbtree, &fcn->node); + + ngx_slab_free_locked(cache->shpool, fcn); + + ngx_shmtx_unlock(&cache->shpool->mutex); + + ngx_create_hashed_filename(path, name, len); + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, + "http file cache expire: \"%s\"", name); + + if (ngx_delete_file(name) == NGX_FILE_ERROR) { + ngx_log_error(NGX_LOG_CRIT, ngx_cycle->log, ngx_errno, + ngx_delete_file_n " \"%s\" failed", name); + } + + ngx_free(name); + + ngx_shmtx_lock(&cache->shpool->mutex); } + + ngx_shmtx_unlock(&cache->shpool->mutex); + + return wait; } -#endif +time_t +ngx_http_file_cache_cleaner(void *data) +{ + ngx_http_file_cache_t *cache = data; + + time_t now, next; + ngx_tree_ctx_t tree; + + now = ngx_time(); + + if (now >= cache->next_clean_time) { + ngx_log_error(NGX_LOG_NOTICE, ngx_cycle->log, 0, + "clean unused cache files"); -ngx_int_t ngx_http_cache_cleaner_handler(ngx_gc_t *gc, ngx_str_t *name, - ngx_dir_t *dir) + tree.init_handler = NULL; + tree.file_handler = ngx_http_file_cache_clean_file; + tree.pre_tree_handler = ngx_http_file_cache_clean_noop; + tree.post_tree_handler = ngx_http_file_cache_clean_noop; + tree.spec_handler = ngx_http_file_cache_clean_file; + tree.data = cache; + tree.alloc = 0; + tree.log = ngx_cycle->log; + + (void) ngx_walk_tree(&tree, &cache->path->name); + + ngx_time_update(0, 0); + + next = ngx_next_time(cache->clean_time); + + if (next == -1) { + ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, ngx_errno, + ngx_next_time_n " failed"); + return 60; + } + + cache->next_clean_time = next; + } + + next = ngx_http_file_cache_expire(cache, 0); + + now = ngx_time(); + + return (now + next < cache->next_clean_time) ? next: + cache->next_clean_time - now; +} + + +static ngx_int_t +ngx_http_file_cache_clean_noop(ngx_tree_ctx_t *ctx, ngx_str_t *path) +{ + return NGX_OK; +} + + +static ngx_int_t +ngx_http_file_cache_clean_file(ngx_tree_ctx_t *ctx, ngx_str_t *path) { - int rc; - ngx_buf_t buf; - ngx_http_cache_t c; - u_char data[sizeof(ngx_http_cache_header_t)]; + u_char *p, key[2 * NGX_HTTP_CACHE_KEY_LEN]; + ngx_int_t n; + ngx_uint_t i; + ngx_http_file_cache_t *cache; + ngx_http_file_cache_node_t *node; + + if (path->len < 2 * NGX_HTTP_CACHE_KEY_LEN) { + goto clean; + } + + p = &path->data[path->len - 2 * NGX_HTTP_CACHE_KEY_LEN]; + + for (i = 0; i < NGX_HTTP_CACHE_KEY_LEN; i++) { + n = ngx_hextoi(p, 2); - ngx_memzero(&c, sizeof(ngx_http_cache_t)); + if (n == NGX_ERROR) { + goto clean; + } + + p += 2; - c.file.fd = NGX_INVALID_FILE; - c.file.name = *name; - c.file.log = gc->log; + key[i] = (u_char) n; + } - c.header_size = sizeof(ngx_http_cache_header_t); - c.buf = &buf; - c.log = gc->log; - c.key_len = 0; + cache = ctx->data; - buf.memory = 1; - buf.temporary = 1; - buf.pos = data; - buf.last = data; - buf.start = data; - buf.end = data + sizeof(ngx_http_cache_header_t); + ngx_shmtx_lock(&cache->shpool->mutex); - rc = ngx_http_file_cache_open(&c); + node = ngx_http_file_cache_lookup(cache, key); - /* TODO: NGX_AGAIN */ + ngx_shmtx_unlock(&cache->shpool->mutex); - if (rc != NGX_ERROR&& rc != NGX_DECLINED && rc != NGX_HTTP_CACHE_STALE) { + if (node) { return NGX_OK; } - if (ngx_delete_file(name->data) == NGX_FILE_ERROR) { - ngx_log_error(NGX_LOG_CRIT, c.log, ngx_errno, - ngx_delete_file_n " \"%s\" failed", name->data); +clean: + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ctx->log, 0, + "http file cache clean: \"%s\"", path->data); + + if (ngx_delete_file(path->data) == NGX_FILE_ERROR) { + ngx_log_error(NGX_LOG_CRIT, ctx->log, ngx_errno, + ngx_delete_file_n " \"%s\" failed", path->data); return NGX_ERROR; } - gc->deleted++; - gc->freed += ngx_de_size(dir); - return NGX_OK; } + + +time_t +ngx_http_file_cache_valid(ngx_array_t *cache_valid, ngx_uint_t status) +{ + ngx_uint_t i; + ngx_http_cache_valid_t *valid; + + valid = cache_valid->elts; + for (i = 0; i < cache_valid->nelts; i++) { + + if (valid[i].status == 0) { + return valid[i].valid; + } + + if (valid[i].status == status) { + return valid[i].valid; + } + } + + return 0; +} + + +char * +ngx_http_file_cache_set_slot(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + u_char *last, *p; + time_t inactive, clean_time, next; + ssize_t size; + ngx_str_t s, name, *value; + ngx_uint_t i, n; + ngx_http_file_cache_t *cache; + + cache = ngx_pcalloc(cf->pool, sizeof(ngx_http_file_cache_t)); + if (cache == NULL) { + return NGX_CONF_ERROR; + } + + cache->path = ngx_pcalloc(cf->pool, sizeof(ngx_path_t)); + if (cache->path == NULL) { + return NGX_CONF_ERROR; + } + + inactive = 600; + clean_time = 5 * 60 * 60; + + name.len = 0; + size = 0; + + value = cf->args->elts; + + cache->path->name = value[1]; + + if (cache->path->name.data[cache->path->name.len - 1] == '/') { + cache->path->name.len--; + } + + if (ngx_conf_full_name(cf->cycle, &cache->path->name, 0) != NGX_OK) { + return NGX_CONF_ERROR; + } + + for (i = 2; i < cf->args->nelts; i++) { + + if (ngx_strncmp(value[i].data, "levels=", 7) == 0) { + + n = 0; + p = value[i].data + 7; + last = value[i].data + value[i].len; + + while (p < last) { + + if (*p > '0' && *p < '6') { + + cache->path->level[n] = *p++ - '0'; + cache->path->len += cache->path->level[n] + 1; + + if (p == last) { + break; + } + + if (*p++ == ':') { + + if (n > 2) { + goto invalid_levels; + } + + if (cache->path->level[n] == 0) { + goto invalid_levels; + } + + n++; + + continue; + } + } + + goto invalid_levels; + } + + if (cache->path->len < 10 + 3) { + continue; + } + + invalid_levels: + + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid \"levels\" \"%V\"", &value[i]); + return NGX_CONF_ERROR; + } + + if (ngx_strncmp(value[i].data, "keys_zone=", 10) == 0) { + + name.data = value[i].data + 10; + + p = (u_char *) ngx_strchr(name.data, ':'); + + if (p) { + name.len = p - name.data; + + p++; + + s.len = value[i].data + value[i].len - p; + s.data = p; + + size = ngx_parse_size(&s); + if (size > 8191) { + continue; + } + } + + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid keys zone size \"%V\"", &value[i]); + return NGX_CONF_ERROR; + } + + if (ngx_strncmp(value[i].data, "inactive=", 9) == 0) { + + s.len = value[i].len - 9; + s.data = value[i].data + 9; + + inactive = ngx_parse_time(&s, 1); + if (inactive < 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid inactive value \"%V\"", &value[i]); + return NGX_CONF_ERROR; + } + + continue; + } + + if (ngx_strncmp(value[i].data, "clean_time=", 11) == 0) { + + s.len = value[i].len - 11; + s.data = value[i].data + 11; + + clean_time = ngx_parse_time(&s, 1); + if (clean_time < 0 || clean_time > 24 * 60 * 60) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid clean_time value \"%V\"", &value[i]); + return NGX_CONF_ERROR; + } + + continue; + } + + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid parameter \"%V\"", &value[i]); + return NGX_CONF_ERROR; + } + + if (name.len == 0 || size == 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "\"%V\" must have \"keys_zone\" parameter", + &cmd->name); + return NGX_CONF_ERROR; + } + + cache->path->cleaner = ngx_http_file_cache_cleaner; + cache->path->data = cache; + + if (ngx_add_path(cf, &cache->path) != NGX_OK) { + return NGX_CONF_ERROR; + } + + cache->shm_zone = ngx_shared_memory_add(cf, &name, size, cmd->post); + if (cache->shm_zone == NULL) { + return NGX_CONF_ERROR; + } + + if (cache->shm_zone->data) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "duplicate zone \"%V\"", &name); + return NGX_CONF_ERROR; + } + + next = ngx_next_time(clean_time); + + if (next == -1) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, ngx_errno, + ngx_next_time_n " failed"); + return NGX_CONF_ERROR; + } + + cache->shm_zone->init = ngx_http_file_cache_init; + cache->shm_zone->data = cache; + + cache->inactive = inactive; + cache->clean_time = clean_time; + cache->next_clean_time = next; + + return NGX_CONF_OK; +} + + +char * +ngx_http_file_cache_valid_set_slot(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf) +{ + char *p = conf; + + time_t valid; + ngx_str_t *value; + ngx_uint_t i, n, status; + ngx_array_t **a; + ngx_http_cache_valid_t *v; + static ngx_uint_t statuses[] = { 200, 301, 302 }; + + a = (ngx_array_t **) (p + cmd->offset); + + if (*a == NGX_CONF_UNSET_PTR) { + *a = ngx_array_create(cf->pool, 1, sizeof(ngx_http_cache_valid_t)); + if (*a == NULL) { + return NGX_CONF_ERROR; + } + } + + value = cf->args->elts; + n = cf->args->nelts - 1; + + valid = ngx_parse_time(&value[n], 1); + if (valid < 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid time value \"%V\"", &value[n]); + return NGX_CONF_ERROR; + } + + if (n == 1) { + + for (i = 0; i < 3; i++) { + v = ngx_array_push(*a); + if (v == NULL) { + return NGX_CONF_ERROR; + } + + v->status = statuses[i]; + v->valid = valid; + } + + return NGX_CONF_OK; + } + + for (i = 1; i < n; i++) { + + if (ngx_strcmp(value[i].data, "any") == 0) { + + status = 0; + + } else { + + status = ngx_atoi(value[i].data, value[i].len); + if (status < 100) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid status \"%V\"", &value[i]); + return NGX_CONF_ERROR; + } + } + + v = ngx_array_push(*a); + if (v == NULL) { + return NGX_CONF_ERROR; + } + + v->status = status; + v->valid = valid; + } + + return NGX_CONF_OK; +} |