diff options
Diffstat (limited to 'nginx')
-rw-r--r-- | nginx/ngx_http_js_module.c | 57 | ||||
-rw-r--r-- | nginx/ngx_js.c | 37 | ||||
-rw-r--r-- | nginx/ngx_js.h | 9 | ||||
-rw-r--r-- | nginx/ngx_js_fetch.c | 21 | ||||
-rw-r--r-- | nginx/ngx_js_shared_dict.c | 1374 | ||||
-rw-r--r-- | nginx/ngx_js_shared_dict.h | 1 | ||||
-rw-r--r-- | nginx/ngx_qjs_fetch.c | 21 | ||||
-rw-r--r-- | nginx/ngx_stream_js_module.c | 53 | ||||
-rw-r--r-- | nginx/t/js_fetch.t | 31 | ||||
-rw-r--r-- | nginx/t/js_fetch_https.t | 10 | ||||
-rw-r--r-- | nginx/t/js_fetch_objects.t | 10 | ||||
-rw-r--r-- | nginx/t/js_fetch_resolver.t | 10 | ||||
-rw-r--r-- | nginx/t/js_fetch_timeout.t | 10 | ||||
-rw-r--r-- | nginx/t/js_fetch_verify.t | 10 | ||||
-rw-r--r-- | nginx/t/js_internal_redirect.t | 29 | ||||
-rw-r--r-- | nginx/t/js_periodic.t | 2 | ||||
-rw-r--r-- | nginx/t/js_periodic_fetch.t | 19 | ||||
-rw-r--r-- | nginx/t/js_shared_dict.t | 10 | ||||
-rw-r--r-- | nginx/t/js_shared_dict_state.t | 256 | ||||
-rw-r--r-- | nginx/t/stream_js_fetch.t | 10 | ||||
-rw-r--r-- | nginx/t/stream_js_fetch_https.t | 10 | ||||
-rw-r--r-- | nginx/t/stream_js_fetch_init.t | 10 | ||||
-rw-r--r-- | nginx/t/stream_js_periodic_fetch.t | 10 | ||||
-rw-r--r-- | nginx/t/stream_js_shared_dict.t | 10 | ||||
-rw-r--r-- | nginx/t/stream_js_shared_dict_state.t | 137 |
25 files changed, 1860 insertions, 297 deletions
diff --git a/nginx/ngx_http_js_module.c b/nginx/ngx_http_js_module.c index 40bb83a5..45ddf17e 100644 --- a/nginx/ngx_http_js_module.c +++ b/nginx/ngx_http_js_module.c @@ -433,6 +433,13 @@ static ngx_command_t ngx_http_js_commands[] = { offsetof(ngx_http_js_loc_conf_t, reuse), NULL }, + { ngx_string("js_context_reuse_max_size"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_conf_set_size_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_js_loc_conf_t, reuse_max_size), + NULL }, + { ngx_string("js_import"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE13, ngx_js_import, @@ -2448,6 +2455,8 @@ ngx_http_js_ext_send_header(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, return NJS_ERROR; } + r->disable_not_modified = 1; + if (ngx_http_send_header(r) == NGX_ERROR) { return NJS_ERROR; } @@ -2731,6 +2740,8 @@ ngx_http_js_ext_return(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, cv.value.data = text.start; cv.value.len = text.length; + r->disable_not_modified = 1; + ctx->status = ngx_http_send_response(r, status, NULL, &cv); if (ctx->status == NGX_ERROR) { @@ -5438,6 +5449,8 @@ ngx_http_qjs_ext_return(JSContext *cx, JSValueConst this_val, cv.value.data = body.data; cv.value.len = body.len; + r->disable_not_modified = 1; + ctx->status = ngx_http_send_response(r, status, NULL, &cv); if (ctx->status == NGX_ERROR) { @@ -5663,6 +5676,8 @@ ngx_http_qjs_ext_send_header(JSContext *cx, JSValueConst this_val, return JS_ThrowInternalError(cx, "failed to set content type"); } + r->disable_not_modified = 1; + if (ngx_http_send_header(r) == NGX_ERROR) { return JS_ThrowInternalError(cx, "failed to send header"); } @@ -7684,21 +7699,12 @@ ngx_http_js_init(ngx_conf_t *cf) static ngx_int_t -ngx_http_js_init_worker(ngx_cycle_t *cycle) +ngx_http_js_init_worker_periodics(ngx_js_main_conf_t *jmcf) { ngx_uint_t i; ngx_js_periodic_t *periodics; - ngx_js_main_conf_t *jmcf; - - if ((ngx_process != NGX_PROCESS_WORKER) - && ngx_process != NGX_PROCESS_SINGLE) - { - return NGX_OK; - } - - jmcf = ngx_http_cycle_get_module_main_conf(cycle, ngx_http_js_module); - if (jmcf == NULL || jmcf->periodics == NULL) { + if (jmcf->periodics == NULL) { return NGX_OK; } @@ -7726,6 +7732,35 @@ ngx_http_js_init_worker(ngx_cycle_t *cycle) } +static ngx_int_t +ngx_http_js_init_worker(ngx_cycle_t *cycle) +{ + ngx_js_main_conf_t *jmcf; + + if ((ngx_process != NGX_PROCESS_WORKER) + && ngx_process != NGX_PROCESS_SINGLE) + { + return NGX_OK; + } + + jmcf = ngx_http_cycle_get_module_main_conf(cycle, ngx_http_js_module); + + if (jmcf == NULL) { + return NGX_OK; + } + + if (ngx_http_js_init_worker_periodics(jmcf) != NGX_OK) { + return NGX_ERROR; + } + + if (ngx_js_dict_init_worker(jmcf) != NGX_OK) { + return NGX_ERROR; + } + + return NGX_OK; +} + + static char * ngx_http_js_periodic(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { diff --git a/nginx/ngx_js.c b/nginx/ngx_js.c index e4bae32a..01d4bb2a 100644 --- a/nginx/ngx_js.c +++ b/nginx/ngx_js.c @@ -31,12 +31,6 @@ typedef struct { } ngx_js_rejected_promise_t; -#if defined(PATH_MAX) -#define NGX_MAX_PATH PATH_MAX -#else -#define NGX_MAX_PATH 4096 -#endif - typedef struct { int fd; njs_str_t name; @@ -984,6 +978,11 @@ ngx_qjs_clone(ngx_js_ctx_t *ctx, ngx_js_loc_conf_t *cf, void *external) "js load module exception: %V", &exception); goto destroy; } + + if (i != length - 1) { + /* JS_EvalFunction() does JS_FreeValue(cx, rv) for the last rv. */ + JS_FreeValue(cx, rv); + } } if (JS_ResolveModule(cx, rv) < 0) { @@ -1133,6 +1132,7 @@ ngx_engine_qjs_destroy(ngx_engine_t *e, ngx_js_ctx_t *ctx, JSRuntime *rt; JSContext *cx; JSClassID class_id; + JSMemoryUsage stats; ngx_qjs_event_t *event; ngx_js_opaque_t *opaque; njs_rbtree_node_t *node; @@ -1198,6 +1198,28 @@ ngx_engine_qjs_destroy(ngx_engine_t *e, ngx_js_ctx_t *ctx, cln->data = conf->reuse_queue; } + /* + * After the request object is freed, the runtime's memory usage should + * be low. It can only remain high if the global scope was + * modified. + * + * To prevent unlimited memory consumption growth, check whether memory + * usage exceeds the configured limit. The check is performed rarely to + * avoid performance impact of JS_ComputeMemoryUsage() which is slow. + */ + + if ((ngx_random() & 0xff) == 1) { + JS_ComputeMemoryUsage(JS_GetRuntime(cx), &stats); + + if ((size_t) stats.malloc_size > conf->reuse_max_size) { + ngx_log_error(NGX_LOG_WARN, ctx->log, 0, + "js remaining memory usage of the context " + "exceeds \"js_context_reuse_max_size\" limit: %L" + ", not reusing it", stats.malloc_size); + goto free_ctx; + } + } + if (ngx_js_queue_push(conf->reuse_queue, cx) != NGX_OK) { goto free_ctx; } @@ -3950,6 +3972,7 @@ ngx_js_create_conf(ngx_conf_t *cf, size_t size) conf->preload_objects = NGX_CONF_UNSET_PTR; conf->reuse = NGX_CONF_UNSET_SIZE; + conf->reuse_max_size = NGX_CONF_UNSET_SIZE; conf->buffer_size = NGX_CONF_UNSET_SIZE; conf->max_response_body_size = NGX_CONF_UNSET_SIZE; conf->timeout = NGX_CONF_UNSET_MSEC; @@ -4059,6 +4082,8 @@ ngx_js_merge_conf(ngx_conf_t *cf, void *parent, void *child, ngx_conf_merge_msec_value(conf->timeout, prev->timeout, 60000); ngx_conf_merge_size_value(conf->reuse, prev->reuse, 128); + ngx_conf_merge_size_value(conf->reuse_max_size, prev->reuse_max_size, + 4 * 1024 * 1024); ngx_conf_merge_size_value(conf->buffer_size, prev->buffer_size, 16384); ngx_conf_merge_size_value(conf->max_response_body_size, prev->max_response_body_size, 1048576); diff --git a/nginx/ngx_js.h b/nginx/ngx_js.h index bb7c1d26..99330f88 100644 --- a/nginx/ngx_js.h +++ b/nginx/ngx_js.h @@ -16,8 +16,6 @@ #include <njs.h> #include <njs_rbtree.h> #include <njs_arr.h> -#include "ngx_js_fetch.h" -#include "ngx_js_shared_dict.h" #if (NJS_HAVE_QUICKJS) #include <qjs.h> @@ -124,6 +122,7 @@ typedef struct { ngx_uint_t type; \ ngx_engine_t *engine; \ ngx_uint_t reuse; \ + size_t reuse_max_size; \ ngx_js_queue_t *reuse_queue; \ ngx_str_t cwd; \ ngx_array_t *imports; \ @@ -317,6 +316,9 @@ ngx_int_t ngx_js_exception(njs_vm_t *vm, ngx_str_t *s); ngx_engine_t *ngx_njs_clone(ngx_js_ctx_t *ctx, ngx_js_loc_conf_t *cf, void *external); +#define NGX_CHB_CTX_INIT(chain, pool) \ + njs_chb_init(chain, pool, (njs_chb_alloc_t) ngx_palloc, NULL) + #if (NJS_HAVE_QUICKJS) typedef struct ngx_qjs_event_s ngx_qjs_event_t; @@ -437,4 +439,7 @@ extern njs_module_t njs_xml_module; extern njs_module_t njs_zlib_module; +#include "ngx_js_fetch.h" +#include "ngx_js_shared_dict.h" + #endif /* _NGX_JS_H_INCLUDED_ */ diff --git a/nginx/ngx_js_fetch.c b/nginx/ngx_js_fetch.c index 45f2dc10..faa38aab 100644 --- a/nginx/ngx_js_fetch.c +++ b/nginx/ngx_js_fetch.c @@ -514,6 +514,7 @@ ngx_js_ext_fetch(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, ngx_url_t u; ngx_uint_t i; njs_bool_t has_host; + ngx_str_t method; ngx_pool_t *pool; njs_value_t *init, *value; ngx_js_http_t *http; @@ -674,6 +675,13 @@ ngx_js_ext_fetch(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, continue; } + if (h[i].key.len == 14 + && ngx_strncasecmp(h[i].key.data, (u_char *) "Content-Length", 14) + == 0) + { + continue; + } + njs_chb_append(&http->chain, h[i].key.data, h[i].key.len); njs_chb_append_literal(&http->chain, ": "); njs_chb_append(&http->chain, h[i].value.data, h[i].value.len); @@ -693,7 +701,18 @@ ngx_js_ext_fetch(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, njs_chb_append(&http->chain, request.body.data, request.body.len); } else { - njs_chb_append_literal(&http->chain, CRLF); + method = request.method; + + if ((method.len == 4 + && (ngx_strncasecmp(method.data, (u_char *) "POST", 4) == 0)) + || (method.len == 3 + && ngx_strncasecmp(method.data, (u_char *) "PUT", 3) == 0)) + { + njs_chb_append_literal(&http->chain, "Content-Length: 0" CRLF CRLF); + + } else { + njs_chb_append_literal(&http->chain, CRLF); + } } if (u.addrs == NULL) { diff --git a/nginx/ngx_js_shared_dict.c b/nginx/ngx_js_shared_dict.c index ccca530d..5445b4f2 100644 --- a/nginx/ngx_js_shared_dict.c +++ b/nginx/ngx_js_shared_dict.c @@ -18,17 +18,10 @@ typedef struct { ngx_rbtree_t rbtree_expire; ngx_rbtree_node_t sentinel_expire; -} ngx_js_dict_sh_t; - -typedef struct { - ngx_str_node_t sn; - ngx_rbtree_node_t expire; - union { - ngx_str_t value; - double number; - } u; -} ngx_js_dict_node_t; + unsigned dirty:1; + unsigned writing:1; +} ngx_js_dict_sh_t; struct ngx_js_dict_s { @@ -36,16 +29,47 @@ struct ngx_js_dict_s { ngx_js_dict_sh_t *sh; ngx_slab_pool_t *shpool; + /** + * in order for ngx_js_dict_t to be used as a ngx_event_t data, + * fd is used for event debug and should be at the same position + * as in ngx_connection_t. see ngx_event_ident() for details. + */ + ngx_socket_t fd; + ngx_msec_t timeout; ngx_flag_t evict; #define NGX_JS_DICT_TYPE_STRING 0 #define NGX_JS_DICT_TYPE_NUMBER 1 ngx_uint_t type; + ngx_event_t save_event; + ngx_str_t state_file; + ngx_str_t state_temp_file; + ngx_js_dict_t *next; }; +typedef union { + ngx_str_t str; /* NGX_JS_DICT_TYPE_STRING */ + double number; /* NGX_JS_DICT_TYPE_NUMBER */ +} ngx_js_dict_value_t; + + +typedef struct { + ngx_str_node_t sn; + ngx_rbtree_node_t expire; + ngx_js_dict_value_t value; +} ngx_js_dict_node_t; + + +typedef struct { + ngx_str_t key; + ngx_js_dict_value_t value; + ngx_msec_t expire; +} ngx_js_dict_entry_t; + + static njs_int_t njs_js_ext_shared_dict_capacity(njs_vm_t *vm, njs_object_prop_t *prop, uint32_t unused, njs_value_t *value, njs_value_t *setval, njs_value_t *retval); @@ -79,25 +103,25 @@ static njs_int_t njs_js_ext_shared_dict_type(njs_vm_t *vm, njs_object_prop_t *prop, uint32_t unused, njs_value_t *value, njs_value_t *setval, njs_value_t *retval); static ngx_js_dict_node_t *ngx_js_dict_lookup(ngx_js_dict_t *dict, - njs_str_t *key); + ngx_str_t *key); #define NGX_JS_DICT_FLAG_MUST_EXIST 1 #define NGX_JS_DICT_FLAG_MUST_NOT_EXIST 2 static ngx_int_t ngx_js_dict_set(njs_vm_t *vm, ngx_js_dict_t *dict, - njs_str_t *key, njs_value_t *value, ngx_msec_t timeout, unsigned flags); + ngx_str_t *key, njs_value_t *value, ngx_msec_t timeout, unsigned flags); static ngx_int_t ngx_js_dict_add(njs_vm_t *vm, ngx_js_dict_t *dict, - njs_str_t *key, njs_value_t *value, ngx_msec_t timeout, ngx_msec_t now); + ngx_str_t *key, njs_value_t *value, ngx_msec_t timeout, ngx_msec_t now); static ngx_int_t ngx_js_dict_update(njs_vm_t *vm, ngx_js_dict_t *dict, ngx_js_dict_node_t *node, njs_value_t *value, ngx_msec_t timeout, ngx_msec_t now); static ngx_int_t ngx_js_dict_get(njs_vm_t *vm, ngx_js_dict_t *dict, - njs_str_t *key, njs_value_t *retval); + ngx_str_t *key, njs_value_t *retval); static ngx_int_t ngx_js_dict_incr(njs_vm_t *vm, ngx_js_dict_t *dict, - njs_str_t *key, njs_value_t *delta, njs_value_t *init, double *value, + ngx_str_t *key, njs_value_t *delta, njs_value_t *init, double *value, ngx_msec_t timeout); static ngx_int_t ngx_js_dict_delete(njs_vm_t *vm, ngx_js_dict_t *dict, - njs_str_t *key, njs_value_t *retval); + ngx_str_t *key, njs_value_t *retval); static ngx_int_t ngx_js_dict_copy_value_locked(njs_vm_t *vm, ngx_js_dict_t *dict, ngx_js_dict_node_t *node, njs_value_t *retval); @@ -622,8 +646,14 @@ njs_js_ext_shared_dict_clear(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, done: + dict->sh->dirty = 1; + ngx_rwlock_unlock(&dict->sh->rwlock); + if (dict->state_file.data && !dict->save_event.timer_set) { + ngx_add_timer(&dict->save_event, 1000); + } + njs_value_undefined_set(retval); return NJS_OK; @@ -662,7 +692,7 @@ njs_js_ext_shared_dict_delete(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, njs_index_t unused, njs_value_t *retval) { ngx_int_t rc; - njs_str_t key; + ngx_str_t key; ngx_shm_zone_t *shm_zone; shm_zone = njs_vm_external(vm, ngx_js_shared_dict_proto_id, @@ -672,7 +702,7 @@ njs_js_ext_shared_dict_delete(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, return NJS_ERROR; } - if (ngx_js_string(vm, njs_arg(args, nargs, 1), &key) != NGX_OK) { + if (ngx_js_ngx_string(vm, njs_arg(args, nargs, 1), &key) != NGX_OK) { return NJS_ERROR; } @@ -689,7 +719,7 @@ njs_js_ext_shared_dict_get(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, njs_index_t unused, njs_value_t *retval) { ngx_int_t rc; - njs_str_t key; + ngx_str_t key; ngx_shm_zone_t *shm_zone; shm_zone = njs_vm_external(vm, ngx_js_shared_dict_proto_id, @@ -699,7 +729,7 @@ njs_js_ext_shared_dict_get(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, return NJS_ERROR; } - if (ngx_js_string(vm, njs_arg(args, nargs, 1), &key) != NGX_OK) { + if (ngx_js_ngx_string(vm, njs_arg(args, nargs, 1), &key) != NGX_OK) { return NJS_ERROR; } @@ -717,7 +747,7 @@ static njs_int_t njs_js_ext_shared_dict_has(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, njs_index_t unused, njs_value_t *retval) { - njs_str_t key; + ngx_str_t key; ngx_msec_t now; ngx_time_t *tp; ngx_js_dict_t *dict; @@ -731,7 +761,7 @@ njs_js_ext_shared_dict_has(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, return NJS_ERROR; } - if (ngx_js_string(vm, njs_arg(args, nargs, 1), &key) != NGX_OK) { + if (ngx_js_ngx_string(vm, njs_arg(args, nargs, 1), &key) != NGX_OK) { return NJS_ERROR; } @@ -851,7 +881,7 @@ njs_js_ext_shared_dict_incr(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, { double value; ngx_int_t rc; - njs_str_t key; + ngx_str_t key; ngx_msec_t timeout; njs_value_t *delta, *init, *timeo; ngx_js_dict_t *dict; @@ -872,7 +902,7 @@ njs_js_ext_shared_dict_incr(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, return NJS_ERROR; } - if (ngx_js_string(vm, njs_arg(args, nargs, 1), &key) != NGX_OK) { + if (ngx_js_ngx_string(vm, njs_arg(args, nargs, 1), &key) != NGX_OK) { return NJS_ERROR; } @@ -1058,7 +1088,7 @@ njs_js_ext_shared_dict_pop(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, njs_index_t unused, njs_value_t *retval) { ngx_int_t rc; - njs_str_t key; + ngx_str_t key; ngx_shm_zone_t *shm_zone; shm_zone = njs_vm_external(vm, ngx_js_shared_dict_proto_id, @@ -1068,7 +1098,7 @@ njs_js_ext_shared_dict_pop(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, return NJS_ERROR; } - if (ngx_js_string(vm, njs_arg(args, nargs, 1), &key) != NGX_OK) { + if (ngx_js_ngx_string(vm, njs_arg(args, nargs, 1), &key) != NGX_OK) { return NJS_ERROR; } @@ -1086,7 +1116,7 @@ static njs_int_t njs_js_ext_shared_dict_set(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, njs_index_t flags, njs_value_t *retval) { - njs_str_t key; + ngx_str_t key; ngx_int_t rc; ngx_msec_t timeout; njs_value_t *value, *timeo; @@ -1100,7 +1130,7 @@ njs_js_ext_shared_dict_set(njs_vm_t *vm, njs_value_t *args, njs_uint_t nargs, return NJS_ERROR; } - if (ngx_js_string(vm, njs_arg(args, nargs, 1), &key) != NGX_OK) { + if (ngx_js_ngx_string(vm, njs_arg(args, nargs, 1), &key) != NGX_OK) { return NJS_ERROR; } @@ -1218,7 +1248,7 @@ njs_js_ext_shared_dict_type(njs_vm_t *vm, njs_object_prop_t *prop, uint32_t unused, njs_value_t *value, njs_value_t *setval, njs_value_t *retval) { - njs_str_t type; + ngx_str_t type; ngx_js_dict_t *dict; ngx_shm_zone_t *shm_zone; @@ -1229,36 +1259,41 @@ njs_js_ext_shared_dict_type(njs_vm_t *vm, njs_object_prop_t *prop, } dict = shm_zone->data; - switch (dict->type) { case NGX_JS_DICT_TYPE_STRING: - type = njs_str_value("string"); + ngx_str_set(&type, "string"); break; default: - type = njs_str_value("number"); + ngx_str_set(&type, "number"); break; } - return njs_vm_value_string_create(vm, retval, type.start, type.length); + return njs_vm_value_string_create(vm, retval, type.data, type.len); +} + + +static njs_int_t +ngx_js_dict_shared_error_name(njs_vm_t *vm, njs_object_prop_t *prop, + uint32_t unused, njs_value_t *value, njs_value_t *setval, + njs_value_t *retval) +{ + return njs_vm_value_string_create(vm, retval, + (u_char *) "SharedMemoryError", 17); } static ngx_js_dict_node_t * -ngx_js_dict_lookup(ngx_js_dict_t *dict, njs_str_t *key) +ngx_js_dict_lookup(ngx_js_dict_t *dict, ngx_str_t *key) { uint32_t hash; - ngx_str_t k; ngx_rbtree_t *rbtree; rbtree = &dict->sh->rbtree; - hash = ngx_crc32_long(key->start, key->length); - - k.data = key->start; - k.len = key->length; + hash = ngx_crc32_long(key->data, key->len); - return (ngx_js_dict_node_t *) ngx_str_rbtree_lookup(rbtree, &k, hash); + return (ngx_js_dict_node_t *) ngx_str_rbtree_lookup(rbtree, key, hash); } @@ -1286,7 +1321,7 @@ ngx_js_dict_node_free(ngx_js_dict_t *dict, ngx_js_dict_node_t *node) shpool = dict->shpool; if (dict->type == NGX_JS_DICT_TYPE_STRING) { - ngx_slab_free_locked(shpool, node->u.value.data); + ngx_slab_free_locked(shpool, node->value.str.data); } ngx_slab_free_locked(shpool, node); @@ -1294,7 +1329,7 @@ ngx_js_dict_node_free(ngx_js_dict_t *dict, ngx_js_dict_node_t *node) static ngx_int_t -ngx_js_dict_set(njs_vm_t *vm, ngx_js_dict_t *dict, njs_str_t *key, +ngx_js_dict_set(njs_vm_t *vm, ngx_js_dict_t *dict, ngx_str_t *key, njs_value_t *value, ngx_msec_t timeout, unsigned flags) { ngx_msec_t now; @@ -1331,8 +1366,14 @@ ngx_js_dict_set(njs_vm_t *vm, ngx_js_dict_t *dict, njs_str_t *key, } } + dict->sh->dirty = 1; + ngx_rwlock_unlock(&dict->sh->rwlock); + if (dict->state_file.data && !dict->save_event.timer_set) { + ngx_add_timer(&dict->save_event, 1000); + } + return NGX_OK; memory_error: @@ -1346,20 +1387,19 @@ memory_error: static ngx_int_t -ngx_js_dict_add(njs_vm_t *vm, ngx_js_dict_t *dict, njs_str_t *key, - njs_value_t *value, ngx_msec_t timeout, ngx_msec_t now) +ngx_js_dict_add_value(ngx_js_dict_t *dict, ngx_str_t *key, + ngx_js_dict_value_t *value, ngx_msec_t timeout, ngx_msec_t now) { size_t n; uint32_t hash; - njs_str_t string; ngx_js_dict_node_t *node; if (dict->timeout) { ngx_js_dict_expire(dict, now); } - n = sizeof(ngx_js_dict_node_t) + key->length; - hash = ngx_crc32_long(key->start, key->length); + n = sizeof(ngx_js_dict_node_t) + key->len; + hash = ngx_crc32_long(key->data, key->len); node = ngx_js_dict_alloc(dict, n); if (node == NULL) { @@ -1369,24 +1409,23 @@ ngx_js_dict_add(njs_vm_t *vm, ngx_js_dict_t *dict, njs_str_t *key, node->sn.str.data = (u_char *) node + sizeof(ngx_js_dict_node_t); if (dict->type == NGX_JS_DICT_TYPE_STRING) { - njs_value_string_get(vm, value, &string); - node->u.value.data = ngx_js_dict_alloc(dict, string.length); - if (node->u.value.data == NULL) { + node->value.str.data = ngx_js_dict_alloc(dict, value->str.len); + if (node->value.str.data == NULL) { ngx_slab_free_locked(dict->shpool, node); return NGX_ERROR; } - ngx_memcpy(node->u.value.data, string.start, string.length); - node->u.value.len = string.length; + ngx_memcpy(node->value.str.data, value->str.data, value->str.len); + node->value.str.len = value->str.len; } else { - node->u.number = njs_value_number(value); + node->value.number = value->number; } node->sn.node.key = hash; - ngx_memcpy(node->sn.str.data, key->start, key->length); - node->sn.str.len = key->length; + ngx_memcpy(node->sn.str.data, key->data, key->len); + node->sn.str.len = key->len; ngx_rbtree_insert(&dict->sh->rbtree, &node->sn.node); @@ -1400,6 +1439,29 @@ ngx_js_dict_add(njs_vm_t *vm, ngx_js_dict_t *dict, njs_str_t *key, static ngx_int_t +ngx_js_dict_add(njs_vm_t *vm, ngx_js_dict_t *dict, ngx_str_t *key, + njs_value_t *value, ngx_msec_t timeout, ngx_msec_t now) +{ + njs_str_t string; + ngx_js_dict_value_t entry; + + if (dict->type == NGX_JS_DICT_TYPE_STRING) { + njs_value_string_get(vm, value, &string); + + entry.str.data = string.start; + entry.str.len = string.length; + + } else { + /* GCC complains about uninitialized entry.str.data. */ + entry.str.data = NULL; + entry.number = njs_value_number(value); + } + + return ngx_js_dict_add_value(dict, key, &entry, timeout, now); +} + + +static ngx_int_t ngx_js_dict_update(njs_vm_t *vm, ngx_js_dict_t *dict, ngx_js_dict_node_t *node, njs_value_t *value, ngx_msec_t timeout, ngx_msec_t now) { @@ -1414,14 +1476,14 @@ ngx_js_dict_update(njs_vm_t *vm, ngx_js_dict_t *dict, ngx_js_dict_node_t *node, return NGX_ERROR; } - ngx_slab_free_locked(dict->shpool, node->u.value.data); + ngx_slab_free_locked(dict->shpool, node->value.str.data); ngx_memcpy(p, string.start, string.length); - node->u.value.data = p; - node->u.value.len = string.length; + node->value.str.data = p; + node->value.str.len = string.length; } else { - node->u.number = njs_value_number(value); + node->value.number = njs_value_number(value); } if (dict->timeout) { @@ -1435,7 +1497,7 @@ ngx_js_dict_update(njs_vm_t *vm, ngx_js_dict_t *dict, ngx_js_dict_node_t *node, static ngx_int_t -ngx_js_dict_delete(njs_vm_t *vm, ngx_js_dict_t *dict, njs_str_t *key, +ngx_js_dict_delete(njs_vm_t *vm, ngx_js_dict_t *dict, ngx_str_t *key, njs_value_t *retval) { ngx_int_t rc; @@ -1475,14 +1537,20 @@ ngx_js_dict_delete(njs_vm_t *vm, ngx_js_dict_t *dict, njs_str_t *key, ngx_js_dict_node_free(dict, node); + dict->sh->dirty = 1; + ngx_rwlock_unlock(&dict->sh->rwlock); + if (dict->state_file.data && !dict->save_event.timer_set) { + ngx_add_timer(&dict->save_event, 1000); + } + return rc; } static ngx_int_t -ngx_js_dict_incr(njs_vm_t *vm, ngx_js_dict_t *dict, njs_str_t *key, +ngx_js_dict_incr(njs_vm_t *vm, ngx_js_dict_t *dict, ngx_str_t *key, njs_value_t *delta, njs_value_t *init, double *value, ngx_msec_t timeout) { ngx_msec_t now; @@ -1507,8 +1575,8 @@ ngx_js_dict_incr(njs_vm_t *vm, ngx_js_dict_t *dict, njs_str_t *key, *value = njs_value_number(init); } else { - node->u.number += njs_value_number(delta); - *value = node->u.number; + node->value.number += njs_value_number(delta); + *value = node->value.number; if (dict->timeout) { ngx_rbtree_delete(&dict->sh->rbtree_expire, &node->expire); @@ -1517,14 +1585,20 @@ ngx_js_dict_incr(njs_vm_t *vm, ngx_js_dict_t *dict, njs_str_t *key, } } + dict->sh->dirty = 1; + ngx_rwlock_unlock(&dict->sh->rwlock); + if (dict->state_file.data && !dict->save_event.timer_set) { + ngx_add_timer(&dict->save_event, 1000); + } + return NGX_OK; } static ngx_int_t -ngx_js_dict_get(njs_vm_t *vm, ngx_js_dict_t *dict, njs_str_t *key, +ngx_js_dict_get(njs_vm_t *vm, ngx_js_dict_t *dict, ngx_str_t *key, njs_value_t *retval) { ngx_int_t rc; @@ -1573,14 +1647,14 @@ ngx_js_dict_copy_value_locked(njs_vm_t *vm, ngx_js_dict_t *dict, type = dict->type; if (type == NGX_JS_DICT_TYPE_STRING) { - ret = njs_vm_value_string_create(vm, retval, node->u.value.data, - node->u.value.len); + ret = njs_vm_value_string_create(vm, retval, node->value.str.data, + node->value.str.len); if (ret != NJS_OK) { return NGX_ERROR; } } else { - njs_value_number_set(retval, node->u.number); + njs_value_number_set(retval, node->value.number); } return NGX_OK; @@ -1657,13 +1731,1013 @@ ngx_js_dict_evict(ngx_js_dict_t *dict, ngx_int_t count) } -static njs_int_t -ngx_js_dict_shared_error_name(njs_vm_t *vm, njs_object_prop_t *prop, - uint32_t unused, njs_value_t *value, njs_value_t *setval, - njs_value_t *retval) +static ngx_int_t +ngx_js_render_string(njs_chb_t *chain, ngx_str_t *str) { - return njs_vm_value_string_create(vm, retval, - (u_char *) "SharedMemoryError", 17); + size_t size; + u_char c, *dst, *dst_end; + const u_char *p, *end; + + static char hex2char[16] = { '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' }; + + p = str->data; + end = p + str->len; + size = str->len + 2; + + dst = njs_chb_reserve(chain, size); + if (dst == NULL) { + return NGX_ERROR; + } + + dst_end = dst + size; + + *dst++ = '\"'; + njs_chb_written(chain, 1); + + while (p < end) { + if (dst_end <= dst + sizeof("\\uXXXX")) { + size = ngx_max(end - p + 1, 6); + dst = njs_chb_reserve(chain, size); + if (dst == NULL) { + return NGX_ERROR; + } + + dst_end = dst + size; + } + + if (*p < ' ' || *p == '\\' || *p == '\"') { + c = (u_char) *p++; + *dst++ = '\\'; + njs_chb_written(chain, 2); + + switch (c) { + case '\\': + *dst++ = '\\'; + break; + case '"': + *dst++ = '\"'; + break; + case '\r': + *dst++ = 'r'; + break; + case '\n': + *dst++ = 'n'; + break; + case '\t': + *dst++ = 't'; + break; + case '\b': + *dst++ = 'b'; + break; + case '\f': + *dst++ = 'f'; + break; + default: + *dst++ = 'u'; + *dst++ = '0'; + *dst++ = '0'; + *dst++ = hex2char[(c & 0xf0) >> 4]; + *dst++ = hex2char[c & 0x0f]; + njs_chb_written(chain, 4); + } + + continue; + } + + dst = njs_utf8_copy(dst, &p, end); + + njs_chb_written(chain, dst - chain->last->pos); + } + + njs_chb_append_literal(chain, "\""); + + return NGX_OK; +} + + +static ngx_int_t +ngx_js_dict_render_json(ngx_js_dict_t *dict, njs_chb_t *chain) +{ + u_char *p, *dst; + size_t len; + ngx_msec_t now; + ngx_time_t *tp; + ngx_rbtree_t *rbtree; + ngx_rbtree_node_t *rn, *next; + ngx_js_dict_node_t *node; + + tp = ngx_timeofday(); + now = tp->sec * 1000 + tp->msec; + + rbtree = &dict->sh->rbtree; + + njs_chb_append_literal(chain,"{"); + + if (rbtree->root == rbtree->sentinel) { + njs_chb_append_literal(chain, "}"); + return NGX_OK; + } + + for (rn = ngx_rbtree_min(rbtree->root, rbtree->sentinel); + rn != NULL; + rn = next) + { + node = (ngx_js_dict_node_t *) rn; + + next = ngx_rbtree_next(rbtree, rn); + + if (dict->timeout && now >= node->expire.key) { + continue; + } + + if (ngx_js_render_string(chain, &node->sn.str) != NGX_OK) { + return NGX_ERROR; + } + + njs_chb_append_literal(chain,":{"); + + if (dict->type == NGX_JS_DICT_TYPE_STRING) { + njs_chb_append_literal(chain,"\"value\":"); + + if (ngx_js_render_string(chain, &node->value.str) != NGX_OK) { + return NGX_ERROR; + } + + } else { + len = sizeof("\"value\":.") + 18 + 6; + dst = njs_chb_reserve(chain, len); + if (dst == NULL) { + return NGX_ERROR; + } + + p = njs_sprintf(dst, dst + len, "\"value\":%.6f", + node->value.number); + njs_chb_written(chain, p - dst); + } + + if (dict->timeout) { + len = sizeof(",\"expire\":1000000000"); + dst = njs_chb_reserve(chain, len); + if (dst == NULL) { + return NGX_ERROR; + } + + p = njs_sprintf(dst, dst + len, ",\"expire\":%ui", + node->expire.key); + njs_chb_written(chain, p - dst); + } + + njs_chb_append_literal(chain, "}"); + + if (next != NULL) { + njs_chb_append_literal(chain, ","); + } + } + + njs_chb_append_literal(chain, "}"); + + return NGX_OK; +} + + +static u_char * +ngx_js_skip_space(u_char *start, u_char *end) +{ + u_char *p; + + for (p = start; p != end; p++) { + + switch (*p) { + case ' ': + case '\t': + case '\r': + case '\n': + continue; + } + + break; + } + + return p; +} + + +static uint32_t +ngx_js_unicode(const u_char *p) +{ + u_char c; + uint32_t utf; + njs_uint_t i; + + utf = 0; + + for (i = 0; i < 4; i++) { + utf <<= 4; + c = p[i] | 0x20; + c -= '0'; + if (c > 9) { + c += '0' - 'a' + 10; + } + + utf |= c; + } + + return utf; +} + + +static u_char * +ngx_js_dict_parse_string(ngx_pool_t *pool, u_char *p, u_char *end, + ngx_str_t *str, const char **err, u_char **at) +{ + u_char ch, *s, *dst, *start, *last; + size_t size, surplus; + uint32_t utf, utf_low; + + enum { + sw_usual = 0, + sw_escape, + sw_encoded1, + sw_encoded2, + sw_encoded3, + sw_encoded4, + } state; + + if (*p != '"') { + *err = "unexpected character, expected '\"'"; + goto error; + } + + start = p + 1; + + dst = NULL; + state = 0; + surplus = 0; + + for (p = start; p < end; p++) { + ch = *p; + + switch (state) { + + case sw_usual: + + if (ch == '"') { + break; + } + + if (ch == '\\') { + state = sw_escape; + continue; + } + + if (ch >= ' ') { + continue; + } + + *err = "Invalid source char"; + goto error; + + case sw_escape: + + switch (ch) { + case '"': + case '\\': + case '/': + case 'n': + case 'r': + case 't': + case 'b': + case 'f': + surplus++; + state = sw_usual; + continue; + + case 'u': + /* + * Basic unicode 6 bytes "\uXXXX" in JSON + * and up to 3 bytes in UTF-8. + * + * Surrogate pair: 12 bytes "\uXXXX\uXXXX" in JSON + * and 3 or 4 bytes in UTF-8. + */ + surplus += 3; + state = sw_encoded1; + continue; + } + + *err = "Invalid escape char"; + goto error; + + case sw_encoded1: + case sw_encoded2: + case sw_encoded3: + case sw_encoded4: + + if ((ch >= '0' && ch <= '9') + || (ch >= 'A' && ch <= 'F') + || (ch >= 'a' && ch <= 'f')) + { + state = (state == sw_encoded4) ? sw_usual : state + 1; + continue; + } + + *err = "Invalid Unicode escape sequence"; + goto error; + } + + break; + } + + if (p == end) { + *err = "unexpected end of input"; + goto error; + } + + /* Points to the ending quote mark. */ + last = p; + + size = last - start - surplus; + + if (surplus != 0) { + p = start; + + dst = ngx_palloc(pool, size); + if (dst == NULL) { + *err = "out of memory"; + goto error; + } + + s = dst; + + do { + ch = *p++; + + if (ch != '\\') { + *s++ = ch; + continue; + } + + ch = *p++; + + switch (ch) { + case '"': + case '\\': + case '/': + *s++ = ch; + continue; + + case 'n': + *s++ = '\n'; + continue; + + case 'r': + *s++ = '\r'; + continue; + + case 't': + *s++ = '\t'; + continue; + + case 'b': + *s++ = '\b'; + continue; + + case 'f': + *s++ = '\f'; + continue; + } + + /* "\uXXXX": Unicode escape sequence. */ + + utf = ngx_js_unicode(p); + p += 4; + + if (njs_surrogate_any(utf)) { + + if (utf > 0xdbff || p[0] != '\\' || p[1] != 'u') { + s = njs_utf8_encode(s, NJS_UNICODE_REPLACEMENT); + continue; + } + + p += 2; + + utf_low = ngx_js_unicode(p); + p += 4; + + if (njs_fast_path(njs_surrogate_trailing(utf_low))) { + utf = njs_surrogate_pair(utf, utf_low); + + } else if (njs_surrogate_leading(utf_low)) { + utf = NJS_UNICODE_REPLACEMENT; + s = njs_utf8_encode(s, NJS_UNICODE_REPLACEMENT); + + } else { + utf = utf_low; + s = njs_utf8_encode(s, NJS_UNICODE_REPLACEMENT); + } + } + + s = njs_utf8_encode(s, utf); + + } while (p != last); + + size = s - dst; + start = dst; + } + + str->data = start; + str->len = size; + + return p + 1; + +error: + + *at = p; + + return NULL; +} + + +static u_char * +ngx_js_dict_parse_entry(ngx_js_dict_t *dict, ngx_pool_t *pool, + ngx_js_dict_entry_t *entry, u_char *buf, u_char *end, const char **err, + u_char **at) +{ + int see_value; + u_char *p, *pp; + double number; + ngx_str_t key, str; + + p = buf; + + if (*p++ != '{') { + *err = "unexpected character, expected '{'"; + goto error; + } + + see_value = 0; + + while (1) { + p = ngx_js_skip_space(p, end); + if (p == end) { + *err = "unexpected end of json"; + goto error; + } + + if (*p == '}') { + break; + } + + p = ngx_js_dict_parse_string(pool, p, end, &key, err, at); + if (p == NULL) { + return NULL; + } + + p = ngx_js_skip_space(p, end); + if (p == end) { + *err = "unexpected end of json"; + goto error; + } + + if (*p++ != ':') { + *err = "unexpected character, expected ':'"; + goto error; + } + + p = ngx_js_skip_space(p, end); + if (p == end) { + *err = "unexpected end of json"; + goto error; + } + + if (*p == '\"') { + p = ngx_js_dict_parse_string(pool, p, end, &str, err, at); + if (p == NULL) { + return NULL; + } + + if (key.len == 5 && ngx_strncmp(key.data, "value", 5) == 0) { + if (dict->type != NGX_JS_DICT_TYPE_STRING) { + *err = "expected string value"; + goto error; + } + + entry->value.str = str; + see_value = 1; + } + + } else { + pp = p; + number = strtod((char *) p, (char **) &p); + if (pp == p) { + *err = "invalid number value"; + goto error; + } + + if (key.len == 5 && ngx_strncmp(key.data, "value", 5) == 0) { + if (dict->type == NGX_JS_DICT_TYPE_STRING) { + *err = "expected number value"; + goto error; + } + + entry->value.number = number; + see_value = 1; + + } else if (key.len == 6 + && ngx_strncmp(key.data, "expire", 6) == 0) + { + entry->expire = number; + } + } + + p = ngx_js_skip_space(p, end); + if (p == end) { + *err = "unexpected end of json"; + goto error; + } + + if (*p == ',') { + p++; + } + } + + if (!see_value) { + *err = "missing value"; + goto error; + } + + return p + 1; + +error: + + *at = p; + + return NULL; +} + + +static ngx_int_t +ngx_js_dict_parse_state(ngx_js_dict_t *dict, ngx_pool_t *pool, + ngx_array_t *entries, u_char *buf, u_char *end) +{ + u_char *p, *at; + const char *err; + ngx_js_dict_entry_t *e; + + /* GCC complains about uninitialized err, at. */ + + err = ""; + at = NULL; + + p = ngx_js_skip_space(buf, end); + if (p == end) { + err = "empty json"; + goto error; + } + + if (*p++ != '{') { + err = "json must start with '{'"; + goto error; + } + + while (1) { + p = ngx_js_skip_space(p, end); + if (p == end) { + err = "unexpected end of json"; + goto error; + } + + if (*p == '}') { + p++; + break; + } + + e = ngx_array_push(entries); + if (e == NULL) { + return NGX_ERROR; + } + + p = ngx_js_dict_parse_string(pool, p, end, &e->key, &err, &at); + if (p == NULL) { + p = at; + goto error; + } + + p = ngx_js_skip_space(p, end); + if (p == end) { + err = "unexpected end of json"; + goto error; + } + + if (*p++ != ':') { + err = "unexpected character, expected ':'"; + goto error; + } + + p = ngx_js_skip_space(p, end); + if (p == end) { + err = "unexpected end of json"; + goto error; + } + + p = ngx_js_dict_parse_entry(dict, pool, e, p, end, &err, &at); + if (p == NULL) { + p = at; + goto error; + } + + p = ngx_js_skip_space(p, end); + if (p == end) { + err = "unexpected end of json"; + goto error; + } + + if (*p == ',') { + p++; + } + } + + p = ngx_js_skip_space(p, end); + + if (p != end) { + err = "unexpected character, expected end of json"; + goto error; + } + + return NGX_OK; + +error: + + ngx_log_error(NGX_LOG_EMERG, dict->shm_zone->shm.log, 0, + "invalid format while loading js_shared_dict_zone \"%V\"" + " from state file \"%s\": %s at offset %z", + &dict->shm_zone->shm.name, dict->state_file.data, err, + p - buf); + + return NGX_ERROR; +} + + +static ngx_int_t +ngx_js_dict_save(ngx_js_dict_t *dict) +{ + + u_char *name; + ngx_int_t rc; + ngx_log_t *log; + njs_chb_t chain; + ngx_file_t file; + ngx_pool_t *pool; + ngx_chain_t *out, *cl, **ll; + njs_chb_node_t *node; + ngx_ext_rename_file_t ext; + + log = dict->shm_zone->shm.log; + + pool = ngx_create_pool(NGX_DEFAULT_POOL_SIZE, log); + if (pool == NULL) { + return NGX_ERROR; + } + + ngx_rwlock_wlock(&dict->sh->rwlock); + + if (!dict->sh->dirty) { + ngx_rwlock_unlock(&dict->sh->rwlock); + ngx_destroy_pool(pool); + return NGX_OK; + } + + if (dict->sh->writing) { + ngx_rwlock_unlock(&dict->sh->rwlock); + ngx_destroy_pool(pool); + return NGX_AGAIN; + } + + ngx_rwlock_downgrade(&dict->sh->rwlock); + + NGX_CHB_CTX_INIT(&chain, pool); + + rc = ngx_js_dict_render_json(dict, &chain); + + if (rc != NGX_OK) { + ngx_rwlock_unlock(&dict->sh->rwlock); + ngx_destroy_pool(pool); + return rc; + } + + dict->sh->writing = 1; + dict->sh->dirty = 0; + + ngx_rwlock_unlock(&dict->sh->rwlock); + + name = dict->state_temp_file.data; + + out = NULL; + ll = &out; + + for (node = chain.nodes; node != NULL; node = node->next) { + cl = ngx_alloc_chain_link(pool); + if (cl == NULL) { + goto error; + } + + cl->buf = ngx_calloc_buf(pool); + if (cl->buf == NULL) { + goto error; + } + + cl->buf->pos = node->start; + cl->buf->last = node->pos; + cl->buf->memory = 1; + cl->buf->last_buf = (node->next == NULL) ? 1 : 0; + + *ll = cl; + ll = &cl->next; + } + + *ll = NULL; + + ngx_memzero(&file, sizeof(ngx_file_t)); + file.name = dict->state_temp_file; + file.log = log; + + file.fd = ngx_open_file(file.name.data, NGX_FILE_WRONLY, NGX_FILE_TRUNCATE, + NGX_FILE_DEFAULT_ACCESS); + + if (file.fd == NGX_INVALID_FILE) { + ngx_log_error(NGX_LOG_ALERT, log, ngx_errno, + ngx_open_file_n " \"%s\" failed", name); + goto error; + } + + rc = ngx_write_chain_to_file(&file, out, 0, pool); + + if (rc == NGX_ERROR) { + ngx_log_error(NGX_LOG_ALERT, log, ngx_errno, + ngx_write_fd_n " \"%s\" failed", file.name.data); + goto error; + } + + if (ngx_close_file(file.fd) == NGX_FILE_ERROR) { + ngx_log_error(NGX_LOG_ALERT, log, ngx_errno, + ngx_close_file_n " \"%s\" failed", file.name.data); + } + + file.fd = NGX_INVALID_FILE; + + ext.access = 0; + ext.time = -1; + ext.create_path = 0; + ext.delete_file = 0; + ext.log = log; + + if (ngx_ext_rename_file(&dict->state_temp_file, &dict->state_file, &ext) + != NGX_OK) + { + goto error; + } + + /* no lock required */ + dict->sh->writing = 0; + ngx_destroy_pool(pool); + + return NGX_OK; + +error: + + if (file.fd != NGX_INVALID_FILE + && ngx_close_file(file.fd) == NGX_FILE_ERROR) + { + ngx_log_error(NGX_LOG_ALERT, log, ngx_errno, + ngx_close_file_n " \"%s\" failed", name); + } + + ngx_destroy_pool(pool); + + /* no lock required */ + dict->sh->writing = 0; + dict->sh->dirty = 1; + + return NGX_ERROR; +} + + +static ngx_int_t +ngx_js_dict_load(ngx_js_dict_t *dict) +{ + off_t size; + u_char *name, *buf; + size_t len; + ssize_t n; + ngx_fd_t fd; + ngx_err_t err; + ngx_int_t rc; + ngx_log_t *log; + ngx_uint_t i; + ngx_msec_t now, expire; + ngx_time_t *tp; + ngx_pool_t *pool; + ngx_array_t data; + ngx_file_info_t fi; + ngx_js_dict_entry_t *entries; + + if (dict->state_file.data == NULL) { + return NGX_OK; + } + + log = dict->shm_zone->shm.log; + + name = dict->state_file.data; + + fd = ngx_open_file(name, NGX_FILE_RDONLY, NGX_FILE_OPEN, 0); + if (fd == NGX_INVALID_FILE) { + err = ngx_errno; + + if (err == NGX_ENOENT || err == NGX_ENOPATH) { + return NGX_OK; + } + + ngx_log_error(NGX_LOG_EMERG, log, err, + ngx_open_file_n " \"%s\" failed", name); + return NGX_ERROR; + } + + if (ngx_fd_info(fd, &fi) == NGX_FILE_ERROR) { + ngx_log_error(NGX_LOG_EMERG, log, ngx_errno, + ngx_fd_info_n " \"%s\" failed", name); + pool = NULL; + goto failed; + } + + size = ngx_file_size(&fi); + + if (size == 0) { + + if (ngx_close_file(fd) == NGX_FILE_ERROR) { + ngx_log_error(NGX_LOG_EMERG, log, ngx_errno, + ngx_close_file_n " \"%s\" failed", name); + } + + return NGX_OK; + } + + pool = ngx_create_pool(NGX_DEFAULT_POOL_SIZE, log); + if (pool == NULL) { + goto failed; + } + + len = size; + + buf = ngx_pnalloc(pool, len); + if (buf == NULL) { + goto failed; + } + + n = ngx_read_fd(fd, buf, len); + + if (n == -1) { + ngx_log_error(NGX_LOG_EMERG, log, ngx_errno, + ngx_read_fd_n " \"%s\" failed", name); + goto failed; + } + + if ((size_t) n != len) { + ngx_log_error(NGX_LOG_EMERG, log, 0, + ngx_read_fd_n " has read only %z of %uz from %s", + n, len, name); + goto failed; + } + + if (ngx_close_file(fd) == NGX_FILE_ERROR) { + ngx_log_error(NGX_LOG_EMERG, log, ngx_errno, + ngx_close_file_n " \"%s\" failed", name); + fd = NGX_INVALID_FILE; + goto failed; + } + + fd = NGX_INVALID_FILE; + + if (ngx_array_init(&data, pool, 4, sizeof(ngx_js_dict_entry_t)) + != NGX_OK) + { + goto failed; + } + + rc = ngx_js_dict_parse_state(dict, pool, &data, buf, buf + len); + + if (rc != NGX_OK) { + goto failed; + } + + entries = data.elts; + + tp = ngx_timeofday(); + now = tp->sec * 1000 + tp->msec; + + for (i = 0; i < data.nelts; i++) { + + if (dict->timeout) { + expire = entries[i].expire; + + if (expire && now >= expire) { + dict->sh->dirty = 1; + continue; + } + + if (expire == 0) { + /* treat state without expire as new */ + expire = now + dict->timeout; + dict->sh->dirty = 1; + } + + } else { + expire = 0; + } + + if (ngx_js_dict_lookup(dict, &entries[i].key) != NULL) { + goto failed; + } + + if (ngx_js_dict_add_value(dict, &entries[i].key, &entries[i].value, + expire, 1) + != NGX_OK) + { + goto failed; + } + } + + ngx_destroy_pool(pool); + + return NGX_OK; + +failed: + + if (fd != NGX_INVALID_FILE && ngx_close_file(fd) == NGX_FILE_ERROR) { + ngx_log_error(NGX_LOG_EMERG, log, ngx_errno, + ngx_close_file_n " \"%s\" failed", name); + } + + if (pool) { + ngx_destroy_pool(pool); + } + + return NGX_ERROR; +} + + +static void +ngx_js_dict_save_handler(ngx_event_t *ev) +{ + ngx_int_t rc; + ngx_js_dict_t *dict; + + dict = ev->data; + + rc = ngx_js_dict_save(dict); + + if (rc == NGX_OK) { + return; + } + + if (rc == NGX_ERROR && (ngx_terminate || ngx_exiting)) { + ngx_log_error(NGX_LOG_ALERT, ev->log, 0, + "failed to save the state of shared dict zone \"%V\"", + &dict->shm_zone->shm.name); + return; + } + + /* NGX_ERROR, NGX_AGAIN */ + + ngx_add_timer(ev, 1000); +} + + +ngx_int_t +ngx_js_dict_init_worker(ngx_js_main_conf_t *jmcf) +{ + ngx_js_dict_t *dict; + + if ((ngx_process != NGX_PROCESS_WORKER || ngx_worker != 0) + && ngx_process != NGX_PROCESS_SINGLE) + { + return NGX_OK; + } + + if (jmcf->dicts == NULL) { + return NGX_OK; + } + + for (dict = jmcf->dicts; dict != NULL; dict = dict->next) { + + if (!dict->sh->dirty || !dict->state_file.data) { + continue; + } + + ngx_add_timer(&dict->save_event, 1000); + } + + return NGX_OK; } @@ -1733,6 +2807,10 @@ ngx_js_dict_init_zone(ngx_shm_zone_t *shm_zone, void *data) ngx_sprintf(dict->shpool->log_ctx, " in js shared zone \"%V\"%Z", &shm_zone->shm.name); + if (ngx_js_dict_load(dict) != NGX_OK) { + return NGX_ERROR; + } + return NGX_OK; } @@ -1745,7 +2823,7 @@ ngx_js_shared_dict_zone(ngx_conf_t *cf, ngx_command_t *cmd, void *conf, u_char *p; ssize_t size; - ngx_str_t *value, name, s; + ngx_str_t *value, name, file, s; ngx_flag_t evict; ngx_msec_t timeout; ngx_uint_t i, type; @@ -1756,6 +2834,7 @@ ngx_js_shared_dict_zone(ngx_conf_t *cf, ngx_command_t *cmd, void *conf, evict = 0; timeout = 0; name.len = 0; + ngx_str_null(&file); type = NGX_JS_DICT_TYPE_STRING; value = cf->args->elts; @@ -1807,6 +2886,17 @@ ngx_js_shared_dict_zone(ngx_conf_t *cf, ngx_command_t *cmd, void *conf, continue; } + if (ngx_strncmp(value[i].data, "state=", 6) == 0) { + file.data = value[i].data + 6; + file.len = value[i].len - 6; + + if (ngx_conf_full_name(cf->cycle, &file, 0) != NGX_OK) { + return NGX_CONF_ERROR; + } + + continue; + } + if (ngx_strncmp(value[i].data, "timeout=", 8) == 0) { s.data = value[i].data + 8; @@ -1880,6 +2970,23 @@ ngx_js_shared_dict_zone(ngx_conf_t *cf, ngx_command_t *cmd, void *conf, dict->timeout = timeout; dict->type = type; + dict->save_event.handler = ngx_js_dict_save_handler; + dict->save_event.data = dict; + dict->save_event.log = &cf->cycle->new_log; + dict->fd = -1; + + if (file.data) { + dict->state_file = file; + + p = ngx_pnalloc(cf->pool, file.len + sizeof(".tmp")); + if (p == NULL) { + return NGX_CONF_ERROR; + } + + dict->state_temp_file.data = p; + dict->state_temp_file.len = ngx_sprintf(p, "%V.tmp%Z", &file) - p - 1; + } + return NGX_CONF_OK; } @@ -2079,8 +3186,14 @@ ngx_qjs_ext_shared_dict_clear(JSContext *cx, JSValueConst this_val, done: + dict->sh->dirty = 1; + ngx_rwlock_unlock(&dict->sh->rwlock); + if (dict->state_file.data && !dict->save_event.timer_set) { + ngx_add_timer(&dict->save_event, 1000); + } + return JS_UNDEFINED; } @@ -2630,13 +3743,13 @@ ngx_qjs_dict_copy_value_locked(JSContext *cx, ngx_js_dict_t *dict, ngx_js_dict_node_t *node) { if (dict->type == NGX_JS_DICT_TYPE_STRING) { - return JS_NewStringLen(cx, (const char *) node->u.value.data, - node->u.value.len); + return JS_NewStringLen(cx, (const char *) node->value.str.data, + node->value.str.len); } /* NGX_JS_DICT_TYPE_NUMBER */ - return JS_NewFloat64(cx, node->u.number); + return JS_NewFloat64(cx, node->value.number); } @@ -2658,64 +3771,31 @@ static ngx_int_t ngx_qjs_dict_add(JSContext *cx, ngx_js_dict_t *dict, ngx_str_t *key, JSValue value, ngx_msec_t timeout, ngx_msec_t now) { - size_t n; - uint32_t hash; - ngx_str_t string; - ngx_js_dict_node_t *node; - - if (dict->timeout) { - ngx_js_dict_expire(dict, now); - } - - n = sizeof(ngx_js_dict_node_t) + key->len; - hash = ngx_crc32_long(key->data, key->len); - - node = ngx_js_dict_alloc(dict, n); - if (node == NULL) { - return NGX_ERROR; - } - - node->sn.str.data = (u_char *) node + sizeof(ngx_js_dict_node_t); + ngx_int_t rc; + ngx_js_dict_value_t entry; if (dict->type == NGX_JS_DICT_TYPE_STRING) { - string.data = (u_char *) JS_ToCStringLen(cx, &string.len, value); - if (string.data == NULL) { - ngx_slab_free_locked(dict->shpool, node); + entry.str.data = (u_char *) JS_ToCStringLen(cx, &entry.str.len, value); + if (entry.str.data == NULL) { return NGX_ERROR; } - node->u.value.data = ngx_js_dict_alloc(dict, string.len); - if (node->u.value.data == NULL) { - ngx_slab_free_locked(dict->shpool, node); - JS_FreeCString(cx, (char *) string.data); - return NGX_ERROR; - } - - ngx_memcpy(node->u.value.data, string.data, string.len); - node->u.value.len = string.len; - - JS_FreeCString(cx, (char *) string.data); - } else { - if (JS_ToFloat64(cx, &node->u.number, value) < 0) { - ngx_slab_free_locked(dict->shpool, node); + /* GCC complains about uninitialized entry.str.data. */ + entry.str.data = NULL; + + if (JS_ToFloat64(cx, &entry.number, value) < 0) { return NGX_ERROR; } } - node->sn.node.key = hash; - - ngx_memcpy(node->sn.str.data, key->data, key->len); - node->sn.str.len = key->len; + rc = ngx_js_dict_add_value(dict, key, &entry, timeout, now); - ngx_rbtree_insert(&dict->sh->rbtree, &node->sn.node); - - if (dict->timeout) { - node->expire.key = now + timeout; - ngx_rbtree_insert(&dict->sh->rbtree_expire, &node->expire); + if (dict->type == NGX_JS_DICT_TYPE_STRING) { + JS_FreeCString(cx, (char *) entry.str.data); } - return NGX_OK; + return rc; } @@ -2760,8 +3840,14 @@ ngx_qjs_dict_delete(JSContext *cx, ngx_js_dict_t *dict, ngx_str_t *key, ngx_js_dict_node_free(dict, node); + dict->sh->dirty = 1; + ngx_rwlock_unlock(&dict->sh->rwlock); + if (dict->state_file.data && !dict->save_event.timer_set) { + ngx_add_timer(&dict->save_event, 1000); + } + return ret; } @@ -2829,8 +3915,8 @@ ngx_qjs_dict_incr(JSContext *cx, ngx_js_dict_t *dict, ngx_str_t *key, } } else { - node->u.number += delta; - value = JS_NewFloat64(cx, node->u.number); + node->value.number += delta; + value = JS_NewFloat64(cx, node->value.number); if (dict->timeout) { ngx_rbtree_delete(&dict->sh->rbtree_expire, &node->expire); @@ -2839,8 +3925,14 @@ ngx_qjs_dict_incr(JSContext *cx, ngx_js_dict_t *dict, ngx_str_t *key, } } + dict->sh->dirty = 1; + ngx_rwlock_unlock(&dict->sh->rwlock); + if (dict->state_file.data && !dict->save_event.timer_set) { + ngx_add_timer(&dict->save_event, 1000); + } + return value; } @@ -2870,24 +3962,30 @@ ngx_qjs_dict_set(JSContext *cx, ngx_js_dict_t *dict, ngx_str_t *key, goto memory_error; } - ngx_rwlock_unlock(&dict->sh->rwlock); + } else { - return JS_TRUE; - } + if (flags & NGX_JS_DICT_FLAG_MUST_NOT_EXIST) { + if (!dict->timeout || now < node->expire.key) { + ngx_rwlock_unlock(&dict->sh->rwlock); + return JS_FALSE; + } + } - if (flags & NGX_JS_DICT_FLAG_MUST_NOT_EXIST) { - if (!dict->timeout || now < node->expire.key) { - ngx_rwlock_unlock(&dict->sh->rwlock); - return JS_FALSE; + if (ngx_qjs_dict_update(cx, dict, node, value, timeout, now) + != NGX_OK) + { + goto memory_error; } } - if (ngx_qjs_dict_update(cx, dict, node, value, timeout, now) != NGX_OK) { - goto memory_error; - } + dict->sh->dirty = 1; ngx_rwlock_unlock(&dict->sh->rwlock); + if (dict->state_file.data && !dict->save_event.timer_set) { + ngx_add_timer(&dict->save_event, 1000); + } + return JS_TRUE; memory_error: @@ -2917,16 +4015,16 @@ ngx_qjs_dict_update(JSContext *cx, ngx_js_dict_t *dict, return NGX_ERROR; } - ngx_slab_free_locked(dict->shpool, node->u.value.data); + ngx_slab_free_locked(dict->shpool, node->value.str.data); ngx_memcpy(p, string.data, string.len); - node->u.value.data = p; - node->u.value.len = string.len; + node->value.str.data = p; + node->value.str.len = string.len; JS_FreeCString(cx, (char *) string.data); } else { - if (JS_ToFloat64(cx, &node->u.number, value) < 0) { + if (JS_ToFloat64(cx, &node->value.number, value) < 0) { return NGX_ERROR; } } diff --git a/nginx/ngx_js_shared_dict.h b/nginx/ngx_js_shared_dict.h index b9c7f967..b082962c 100644 --- a/nginx/ngx_js_shared_dict.h +++ b/nginx/ngx_js_shared_dict.h @@ -13,6 +13,7 @@ njs_int_t njs_js_ext_global_shared_prop(njs_vm_t *vm, njs_object_prop_t *prop, njs_value_t *retval); njs_int_t njs_js_ext_global_shared_keys(njs_vm_t *vm, njs_value_t *value, njs_value_t *keys); +ngx_int_t ngx_js_dict_init_worker(ngx_js_main_conf_t *jmcf); extern njs_module_t ngx_js_shared_dict_module; diff --git a/nginx/ngx_qjs_fetch.c b/nginx/ngx_qjs_fetch.c index 084162ba..5ed8fc30 100644 --- a/nginx/ngx_qjs_fetch.c +++ b/nginx/ngx_qjs_fetch.c @@ -241,6 +241,7 @@ ngx_qjs_ext_fetch(JSContext *cx, JSValueConst this_val, int argc, JSValue init, value, promise; ngx_int_t rc; ngx_url_t u; + ngx_str_t method; ngx_uint_t i; ngx_pool_t *pool; ngx_js_ctx_t *ctx; @@ -410,6 +411,13 @@ ngx_qjs_ext_fetch(JSContext *cx, JSValueConst this_val, int argc, continue; } + if (h[i].key.len == 14 + && ngx_strncasecmp(h[i].key.data, (u_char *) "Content-Length", 14) + == 0) + { + continue; + } + njs_chb_append(&http->chain, h[i].key.data, h[i].key.len); njs_chb_append_literal(&http->chain, ": "); njs_chb_append(&http->chain, h[i].value.data, h[i].value.len); @@ -429,7 +437,18 @@ ngx_qjs_ext_fetch(JSContext *cx, JSValueConst this_val, int argc, njs_chb_append(&http->chain, request.body.data, request.body.len); } else { - njs_chb_append_literal(&http->chain, CRLF); + method = request.method; + + if ((method.len == 4 + && (ngx_strncasecmp(method.data, (u_char *) "POST", 4) == 0)) + || (method.len == 3 + && ngx_strncasecmp(method.data, (u_char *) "PUT", 3) == 0)) + { + njs_chb_append_literal(&http->chain, "Content-Length: 0" CRLF CRLF); + + } else { + njs_chb_append_literal(&http->chain, CRLF); + } } if (u.addrs == NULL) { diff --git a/nginx/ngx_stream_js_module.c b/nginx/ngx_stream_js_module.c index 0e022eb0..328ce581 100644 --- a/nginx/ngx_stream_js_module.c +++ b/nginx/ngx_stream_js_module.c @@ -264,6 +264,13 @@ static ngx_command_t ngx_stream_js_commands[] = { offsetof(ngx_stream_js_srv_conf_t, reuse), NULL }, + { ngx_string("js_context_reuse_max_size"), + NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE1, + ngx_conf_set_size_slot, + NGX_STREAM_SRV_CONF_OFFSET, + offsetof(ngx_stream_js_srv_conf_t, reuse_max_size), + NULL }, + { ngx_string("js_import"), NGX_STREAM_MAIN_CONF|NGX_STREAM_SRV_CONF|NGX_CONF_TAKE13, ngx_js_import, @@ -3192,21 +3199,12 @@ ngx_stream_js_periodic_init(ngx_js_periodic_t *periodic) static ngx_int_t -ngx_stream_js_init_worker(ngx_cycle_t *cycle) +ngx_stream_js_init_worker_periodics(ngx_js_main_conf_t *jmcf) { - ngx_uint_t i; - ngx_js_periodic_t *periodics; - ngx_js_main_conf_t *jmcf; - - if ((ngx_process != NGX_PROCESS_WORKER) - && ngx_process != NGX_PROCESS_SINGLE) - { - return NGX_OK; - } + ngx_uint_t i; + ngx_js_periodic_t *periodics; - jmcf = ngx_stream_cycle_get_module_main_conf(cycle, ngx_stream_js_module); - - if (jmcf == NULL || jmcf->periodics == NULL) { + if (jmcf->periodics == NULL) { return NGX_OK; } @@ -3234,6 +3232,35 @@ ngx_stream_js_init_worker(ngx_cycle_t *cycle) } +static ngx_int_t +ngx_stream_js_init_worker(ngx_cycle_t *cycle) +{ + ngx_js_main_conf_t *jmcf; + + if ((ngx_process != NGX_PROCESS_WORKER) + && ngx_process != NGX_PROCESS_SINGLE) + { + return NGX_OK; + } + + jmcf = ngx_stream_cycle_get_module_main_conf(cycle, ngx_stream_js_module); + + if (jmcf == NULL) { + return NGX_OK; + } + + if (ngx_stream_js_init_worker_periodics(jmcf) != NGX_OK) { + return NGX_ERROR; + } + + if (ngx_js_dict_init_worker(jmcf) != NGX_OK) { + return NGX_ERROR; + } + + return NGX_OK; +} + + static char * ngx_stream_js_periodic(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { diff --git a/nginx/t/js_fetch.t b/nginx/t/js_fetch.t index 7ee1a602..76d9238d 100644 --- a/nginx/t/js_fetch.t +++ b/nginx/t/js_fetch.t @@ -52,10 +52,6 @@ http { js_content test.njs; } - location /engine { - js_content test.engine; - } - location /broken { js_content test.broken; } @@ -68,6 +64,10 @@ http { js_content test.body; } + location /body_content_length { + js_content test.body_content_length; + } + location /body_special { js_content test.body_special; } @@ -138,10 +138,6 @@ $t->write_file('test.js', <<EOF); r.return(200, njs.version); } - function engine(r) { - r.return(200, njs.engine); - } - function body(r) { var loc = r.args.loc; var getter = r.args.getter; @@ -164,6 +160,13 @@ $t->write_file('test.js', <<EOF); .catch(e => r.return(501, e.message)) } + async function body_content_length(r) { + let resp = await ngx.fetch(`http://127.0.0.1:$p0/loc`, + {headers: {'Content-Length': '100'}, + body: "CONTENT-BODY"}); + r.return(resp.status); + } + function property(r) { var opts = {headers:{}}; @@ -408,12 +411,12 @@ $t->write_file('test.js', <<EOF); export default {njs: test_njs, body, broken, broken_response, body_special, chain, chunked_ok, chunked_fail, header, header_iter, - host_header, multi, loc, property, engine}; + host_header, multi, loc, property, body_content_length }; EOF $t->try_run('no njs.fetch'); -$t->plan(37); +$t->plan(38); $t->run_daemon(\&http_daemon, port(8082)); $t->waitforsocket('127.0.0.1:' . port(8082)); @@ -516,6 +519,14 @@ like(http_get('/body_special?loc=head/large&method=HEAD'), qr/200 OK.*<empty>$/s, 'fetch head method large content-length'); } +TODO: { +local $TODO = 'not yet' unless has_version('0.9.1'); + +like(http_get('/body_content_length'), qr/200 OK/s, + 'fetch body content-length'); + +} + ############################################################################### sub has_version { diff --git a/nginx/t/js_fetch_https.t b/nginx/t/js_fetch_https.t index 8ede1048..42b5acbb 100644 --- a/nginx/t/js_fetch_https.t +++ b/nginx/t/js_fetch_https.t @@ -48,10 +48,6 @@ http { js_content test.njs; } - location /engine { - js_content test.engine; - } - location /https { js_content test.https; } @@ -106,10 +102,6 @@ $t->write_file('test.js', <<EOF); r.return(200, njs.version); } - function engine(r) { - r.return(200, njs.engine); - } - function https(r) { var url = `https://\${r.args.domain}:$p1/loc`; var opt = {}; @@ -124,7 +116,7 @@ $t->write_file('test.js', <<EOF); .catch(e => r.return(501, e.message)) } - export default {njs: test_njs, https, engine}; + export default {njs: test_njs, https}; EOF my $d = $t->testdir(); diff --git a/nginx/t/js_fetch_objects.t b/nginx/t/js_fetch_objects.t index bc5cc7ed..c9d04c49 100644 --- a/nginx/t/js_fetch_objects.t +++ b/nginx/t/js_fetch_objects.t @@ -45,10 +45,6 @@ http { js_content test.njs; } - location /engine { - js_content test.engine; - } - location /headers { js_content test.headers; } @@ -92,10 +88,6 @@ $t->write_file('test.js', <<EOF); r.return(200, njs.version); } - function engine(r) { - r.return(200, njs.engine); - } - function header(r) { r.return(200, r.headersIn.a); } @@ -528,7 +520,7 @@ $t->write_file('test.js', <<EOF); run(r, tests); } - export default {njs: test_njs, engine, body, headers, request, response, + export default {njs: test_njs, body, headers, request, response, fetch, fetch_multi_header}; EOF diff --git a/nginx/t/js_fetch_resolver.t b/nginx/t/js_fetch_resolver.t index 031ff43c..67680283 100644 --- a/nginx/t/js_fetch_resolver.t +++ b/nginx/t/js_fetch_resolver.t @@ -50,10 +50,6 @@ http { js_content test.njs; } - location /engine { - js_content test.engine; - } - location /dns { js_content test.dns; @@ -108,10 +104,6 @@ $t->write_file('test.js', <<EOF); r.return(200, njs.version); } - function engine(r) { - r.return(200, njs.engine); - } - const p0 = $p0; const p1 = $p1; @@ -141,7 +133,7 @@ $t->write_file('test.js', <<EOF); r.return(c, `\${v.host}:\${v.request_method}:\${foo}:\${bar}:\${body}`); } - export default {njs: test_njs, dns, loc, engine}; + export default {njs: test_njs, dns, loc}; EOF $t->try_run('no njs.fetch'); diff --git a/nginx/t/js_fetch_timeout.t b/nginx/t/js_fetch_timeout.t index ab1ba24a..2ca1510f 100644 --- a/nginx/t/js_fetch_timeout.t +++ b/nginx/t/js_fetch_timeout.t @@ -47,10 +47,6 @@ http { js_content test.njs; } - location /engine { - js_content test.engine; - } - location /normal_timeout { js_content test.timeout_test; } @@ -84,10 +80,6 @@ $t->write_file('test.js', <<EOF); r.return(200, njs.version); } - function engine(r) { - r.return(200, njs.engine); - } - async function timeout_test(r) { let rs = await Promise.allSettled([ 'http://127.0.0.1:$p1/normal_reply', @@ -110,7 +102,7 @@ $t->write_file('test.js', <<EOF); setTimeout((r) => { r.return(200); }, 250, r, 0); } - export default {njs: test_njs, engine, timeout_test, normal_reply, + export default {njs: test_njs, timeout_test, normal_reply, delayed_reply}; EOF diff --git a/nginx/t/js_fetch_verify.t b/nginx/t/js_fetch_verify.t index f98b4d8c..8b691a74 100644 --- a/nginx/t/js_fetch_verify.t +++ b/nginx/t/js_fetch_verify.t @@ -48,10 +48,6 @@ http { js_content test.njs; } - location /engine { - js_content test.engine; - } - location /https { js_content test.https; } @@ -80,10 +76,6 @@ $t->write_file('test.js', <<EOF); r.return(200, njs.version); } - function engine(r) { - r.return(200, njs.engine); - } - function https(r) { ngx.fetch(`https://example.com:$p1/loc`) .then(reply => reply.text()) @@ -91,7 +83,7 @@ $t->write_file('test.js', <<EOF); .catch(e => r.return(501, e.message)); } - export default {njs: test_njs, engine, https}; + export default {njs: test_njs, https}; EOF $t->write_file('openssl.conf', <<EOF); diff --git a/nginx/t/js_internal_redirect.t b/nginx/t/js_internal_redirect.t index abfe79f9..721113bb 100644 --- a/nginx/t/js_internal_redirect.t +++ b/nginx/t/js_internal_redirect.t @@ -11,6 +11,7 @@ use warnings; use strict; use Test::More; +use Socket qw/ CRLF /; BEGIN { use FindBin; chdir($FindBin::Bin); } @@ -54,6 +55,10 @@ http { return 200 redirect$arg_b; } + location /destroyed_ctx { + js_content test.destroyed_ctx; + } + location @named { return 200 named; } @@ -87,7 +92,16 @@ $t->write_file('test.js', <<EOF); } } - export default {njs:test_njs, redirect}; + function destroyed_ctx(r) { + try { + r.return(200); + + } catch (e) { + r.internalRedirect("\@sub"); + } + } + + export default {njs:test_njs, redirect, destroyed_ctx}; EOF @@ -103,5 +117,18 @@ like(http_get('/test?unsafe=1'), qr/500 Internal Server/s, 'unsafe redirect'); like(http_get('/test?quoted=1'), qr/200 .*redirect/s, 'quoted redirect'); +get('/destroyed_ctx', 'If-Match: tt'); ############################################################################### + +sub get { + my ($url, @headers) = @_; + return http( + "GET $url HTTP/1.1" . CRLF . + 'Host: localhost' . CRLF . + 'Connection: close' . CRLF . + join(CRLF, @headers) . CRLF . CRLF + ); +} + +################################################################################ diff --git a/nginx/t/js_periodic.t b/nginx/t/js_periodic.t index d6868935..7e134588 100644 --- a/nginx/t/js_periodic.t +++ b/nginx/t/js_periodic.t @@ -56,7 +56,7 @@ http { server_name localhost; location @periodic { - js_periodic test.tick interval=30ms jitter=1ms; + js_periodic test.tick interval=20ms jitter=1ms; js_periodic test.timer interval=1s worker_affinity=all; js_periodic test.overrun interval=30ms; js_periodic test.affinity interval=50ms worker_affinity=0101; diff --git a/nginx/t/js_periodic_fetch.t b/nginx/t/js_periodic_fetch.t index 0231b662..39385132 100644 --- a/nginx/t/js_periodic_fetch.t +++ b/nginx/t/js_periodic_fetch.t @@ -54,10 +54,6 @@ http { js_periodic test.fetch_exception interval=1s; } - location /engine { - js_content test.engine; - } - location /fetch_ok { return 200 'ok'; } @@ -81,10 +77,6 @@ EOF my $p0 = port(8080); $t->write_file('test.js', <<EOF); - function engine(r) { - r.return(200, njs.engine); - } - async function fetch() { let reply = await ngx.fetch('http://127.0.0.1:$p0/fetch_ok'); let body = await reply.text(); @@ -107,16 +99,15 @@ $t->write_file('test.js', <<EOF); } function test_fetch(r) { - r.return(200, ngx.shared.strings.get('fetch').startsWith('okok')); + r.return(200, ngx.shared.strings.get('fetch')); } function test_multiple_fetches(r) { - r.return(200, ngx.shared.strings.get('multiple_fetches') - .startsWith('ok\@foo')); + r.return(200, ngx.shared.strings.get('multiple_fetches')); } export default { fetch, fetch_exception, multiple_fetches, test_fetch, - test_multiple_fetches, engine }; + test_multiple_fetches }; EOF $t->try_run('no js_periodic with fetch'); @@ -127,8 +118,8 @@ $t->plan(3); select undef, undef, undef, 0.1; -like(http_get('/test_fetch'), qr/true/, 'periodic fetch test'); -like(http_get('/test_multiple_fetches'), qr/true/, 'multiple fetch test'); +like(http_get('/test_fetch'), qr/(ok)+/, 'periodic fetch test'); +like(http_get('/test_multiple_fetches'), qr/ok\@foo/, 'multiple fetch test'); $t->stop(); diff --git a/nginx/t/js_shared_dict.t b/nginx/t/js_shared_dict.t index 8be2831f..b27a33ef 100644 --- a/nginx/t/js_shared_dict.t +++ b/nginx/t/js_shared_dict.t @@ -52,10 +52,6 @@ http { js_content test.njs; } - location /engine { - js_content test.engine; - } - location /add { js_content test.add; } @@ -141,10 +137,6 @@ $t->write_file('test.js', <<'EOF'); r.return(200, njs.version); } - function engine(r) { - r.return(200, njs.engine); - } - function convertToValue(dict, v) { if (dict.type == 'number') { return parseInt(v); @@ -337,7 +329,7 @@ $t->write_file('test.js', <<'EOF'); export default { add, capacity, chain, clear, del, free_space, get, has, incr, items, keys, name, njs: test_njs, pop, replace, set, - set_clear, size, zones, engine, overflow }; + set_clear, size, zones, overflow }; EOF $t->try_run('no js_shared_dict_zone'); diff --git a/nginx/t/js_shared_dict_state.t b/nginx/t/js_shared_dict_state.t new file mode 100644 index 00000000..32eef948 --- /dev/null +++ b/nginx/t/js_shared_dict_state.t @@ -0,0 +1,256 @@ +#!/usr/bin/perl + +# (C) Dmitry Volyntsev +# (C) Nginx, Inc. + +# Tests for js_shared_dict_zone directive, state= parameter. + +############################################################################### + +use warnings; +use strict; + +use Test::More; +use Socket qw/ CRLF /; + +BEGIN { use FindBin; chdir($FindBin::Bin); } + +use lib 'lib'; +use Test::Nginx; + +############################################################################### + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +eval { require JSON::PP; }; +plan(skip_all => "JSON::PP not installed") if $@; + +my $t = Test::Nginx->new()->has(qw/http/) + ->write_file_expand('nginx.conf', <<'EOF'); + +%%TEST_GLOBALS%% + +daemon off; + +events { +} + +http { + %%TEST_GLOBALS_HTTP%% + + js_import test.js; + + js_shared_dict_zone zone=bar:64k type=string state=bar.json; + js_shared_dict_zone zone=waka:32k timeout=1000s type=number state=waka.json; + + server { + listen 127.0.0.1:8080; + server_name localhost; + + location /add { + js_content test.add; + } + + location /clear { + js_content test.clear; + } + + location /delete { + js_content test.del; + } + + location /get { + js_content test.get; + } + + location /incr { + js_content test.incr; + } + + location /pop { + js_content test.pop; + } + + location /set { + js_content test.set; + } + } +} + +EOF + +$t->write_file('bar.json', <<EOF); +{"waka":{"value":"foo","expire":0}, + "bar": { "value" :"\\u0061\\u0062\\u0063"}, +"FOO \\n": { "value" : "BAZ", "unexpected_str": "u\\r" }, + "X": { "valu\\u0065" : "\\n" , "unexpected_num": 23.1 } , + "\\u0061\\u0062\\u0063": { "value" : "def" } , +} +EOF + +$t->write_file('test.js', <<'EOF'); + function convertToValue(dict, v) { + if (dict.type == 'number') { + return parseInt(v); + + } else if (v == 'empty') { + v = ''; + } + + return v; + } + + function add(r) { + var dict = ngx.shared[r.args.dict]; + var value = convertToValue(dict, r.args.value); + + if (r.args.timeout) { + var timeout = Number(r.args.timeout); + r.return(200, dict.add(r.args.key, value, timeout)); + + } else { + r.return(200, dict.add(r.args.key, value)); + } + } + + function clear(r) { + var dict = ngx.shared[r.args.dict]; + var result = dict.clear(); + r.return(200, result === undefined ? 'undefined' : result); + } + + function del(r) { + var dict = ngx.shared[r.args.dict]; + r.return(200, dict.delete(r.args.key)); + } + + function get(r) { + var dict = ngx.shared[r.args.dict]; + var val = dict.get(r.args.key); + + if (val == '') { + val = 'empty'; + + } else if (val === undefined) { + val = 'undefined'; + } + + r.return(200, val); + } + + function incr(r) { + var dict = ngx.shared[r.args.dict]; + var def = r.args.def ? parseInt(r.args.def) : 0; + + if (r.args.timeout) { + var timeout = Number(r.args.timeout); + var val = dict.incr(r.args.key, parseInt(r.args.by), def, timeout); + r.return(200, val); + + } else { + var val = dict.incr(r.args.key, parseInt(r.args.by), def); + r.return(200, val); + } + } + + function pop(r) { + var dict = ngx.shared[r.args.dict]; + var val = dict.pop(r.args.key); + if (val == '') { + val = 'empty'; + + } else if (val === undefined) { + val = 'undefined'; + } + + r.return(200, val); + } + + function set(r) { + var dict = ngx.shared[r.args.dict]; + var value = convertToValue(dict, r.args.value); + + if (r.args.timeout) { + var timeout = Number(r.args.timeout); + r.return(200, dict.set(r.args.key, value, timeout) === dict); + + } else { + r.return(200, dict.set(r.args.key, value) === dict); + } + } + + export default { add, clear, del, get, incr, pop, set }; +EOF + +$t->try_run('no js_shared_dict_zone with state=')->plan(11); + +############################################################################### + +like(http_get('/get?dict=bar&key=waka'), qr/foo/, 'get bar.waka'); +like(http_get('/get?dict=bar&key=bar'), qr/abc/, 'get bar.bar'); +like(http_get('/get?dict=bar&key=FOO%20%0A'), qr/BAZ/, 'get bar["FOO \\n"]'); +like(http_get('/get?dict=bar&key=abc'), qr/def/, 'get bar.abc'); + +http_get('/set?dict=bar&key=waka&value=foo2'); +http_get('/delete?dict=bar&key=bar'); + +http_get('/set?dict=waka&key=foo&value=42'); + +select undef, undef, undef, 1.1; + +$t->reload(); + +my $bar_state = read_state($t, 'bar.json'); +my $waka_state = read_state($t, 'waka.json'); + +is($bar_state->{waka}->{value}, 'foo2', 'get bar.waka from state'); +is($bar_state->{bar}, undef, 'no bar.bar in state'); +is($waka_state->{foo}->{value}, '42', 'get waka.foo from state'); +like($waka_state->{foo}->{expire}, qr/^\d+$/, 'waka.foo expire'); + +http_get('/pop?dict=bar&key=FOO%20%0A'); + +http_get('/incr?dict=waka&key=foo&by=1'); + +select undef, undef, undef, 1.1; + +$bar_state = read_state($t, 'bar.json'); +$waka_state = read_state($t, 'waka.json'); + +is($bar_state->{'FOO \\n'}, undef, 'no bar.FOO \\n in state'); +is($waka_state->{foo}->{value}, '43', 'get waka.foo from state'); + +http_get('/clear?dict=bar'); + +select undef, undef, undef, 1.1; + +$bar_state = read_state($t, 'bar.json'); + +is($bar_state->{waka}, undef, 'no bar.waka in state'); + +############################################################################### + +sub decode_json { + my $json; + eval { $json = JSON::PP::decode_json(shift) }; + + if ($@) { + return "<failed to parse JSON>"; + } + + return $json; +} + +sub read_state { + my ($self, $file) = @_; + my $json = $self->read_file($file); + + if ($json) { + $json = decode_json($json); + } + + return $json; +} + +############################################################################### diff --git a/nginx/t/stream_js_fetch.t b/nginx/t/stream_js_fetch.t index 9a42ae29..cb87eec7 100644 --- a/nginx/t/stream_js_fetch.t +++ b/nginx/t/stream_js_fetch.t @@ -46,10 +46,6 @@ http { js_content test.njs; } - location /engine { - js_content test.engine; - } - location /validate { js_content test.validate; } @@ -103,10 +99,6 @@ $t->write_file('test.js', <<EOF); r.return(200, njs.version); } - function engine(r) { - r.return(200, njs.engine); - } - function validate(r) { r.return((r.requestText == 'QZ') ? 200 : 403); } @@ -166,7 +158,7 @@ $t->write_file('test.js', <<EOF); } export default {njs: test_njs, validate, preread_verify, filter_verify, - access_ok, access_nok, engine}; + access_ok, access_nok}; EOF $t->try_run('no stream njs available'); diff --git a/nginx/t/stream_js_fetch_https.t b/nginx/t/stream_js_fetch_https.t index 987a896a..f397ea70 100644 --- a/nginx/t/stream_js_fetch_https.t +++ b/nginx/t/stream_js_fetch_https.t @@ -47,10 +47,6 @@ http { location /njs { js_content test.njs; } - - location /engine { - js_content test.engine; - } } server { @@ -163,10 +159,6 @@ $t->write_file('test.js', <<EOF); r.return(200, njs.version); } - function engine(r) { - r.return(200, njs.engine); - } - function preread(s) { s.on('upload', function (data, flags) { if (data.startsWith('GO')) { @@ -201,7 +193,7 @@ $t->write_file('test.js', <<EOF); (r.status == 200) ? s.allow(): s.deny(); } - export default {njs: test_njs, engine, preread, access_ok, access_nok}; + export default {njs: test_njs, preread, access_ok, access_nok}; EOF my $d = $t->testdir(); diff --git a/nginx/t/stream_js_fetch_init.t b/nginx/t/stream_js_fetch_init.t index f48b9d5e..3980a9ee 100644 --- a/nginx/t/stream_js_fetch_init.t +++ b/nginx/t/stream_js_fetch_init.t @@ -58,10 +58,6 @@ http { js_content test.njs; } - location /engine { - js_content test.engine; - } - location /success { return 200; } @@ -77,17 +73,13 @@ $t->write_file('test.js', <<EOF); r.return(200, njs.version); } - function engine(r) { - r.return(200, njs.engine); - } - async function access_ok(s) { let reply = await ngx.fetch('http://127.0.0.1:$p/success'); (reply.status == 200) ? s.allow(): s.deny(); } - export default {njs: test_njs, engine, access_ok}; + export default {njs: test_njs, access_ok}; EOF $t->try_run('no stream njs available'); diff --git a/nginx/t/stream_js_periodic_fetch.t b/nginx/t/stream_js_periodic_fetch.t index 60599423..4ebec96e 100644 --- a/nginx/t/stream_js_periodic_fetch.t +++ b/nginx/t/stream_js_periodic_fetch.t @@ -67,10 +67,6 @@ http { listen 127.0.0.1:8080; server_name localhost; - location /engine { - js_content test.engine; - } - location /fetch_ok { return 200 'ok'; } @@ -86,10 +82,6 @@ EOF my $p1 = port(8080); $t->write_file('test.js', <<EOF); - function engine(r) { - r.return(200, njs.engine); - } - async function fetch() { let reply = await ngx.fetch('http://127.0.0.1:$p1/fetch_ok'); let body = await reply.text(); @@ -142,7 +134,7 @@ $t->write_file('test.js', <<EOF); }); } - export default { engine, fetch, fetch_exception, test, multiple_fetches }; + export default { fetch, fetch_exception, test, multiple_fetches }; EOF $t->run_daemon(\&stream_daemon, port(8090)); diff --git a/nginx/t/stream_js_shared_dict.t b/nginx/t/stream_js_shared_dict.t index 915cc40b..0435033d 100644 --- a/nginx/t/stream_js_shared_dict.t +++ b/nginx/t/stream_js_shared_dict.t @@ -43,10 +43,6 @@ http { location / { return 200; } - - location /engine { - js_content test.engine; - } } } @@ -75,10 +71,6 @@ EOF $t->write_file('test.js', <<EOF); import qs from 'querystring'; - function engine(r) { - r.return(200, 'engine'); - } - function preread_verify(s) { var collect = Buffer.from([]); @@ -121,7 +113,7 @@ $t->write_file('test.js', <<EOF); }); } - export default { engine, preread_verify, control_access }; + export default { preread_verify, control_access }; EOF diff --git a/nginx/t/stream_js_shared_dict_state.t b/nginx/t/stream_js_shared_dict_state.t new file mode 100644 index 00000000..c2edb63e --- /dev/null +++ b/nginx/t/stream_js_shared_dict_state.t @@ -0,0 +1,137 @@ +#!/usr/bin/perl + +# (C) Dmitry Volyntsev +# (C) Nginx, Inc. + +# Tests for js_shared_dict_zone directive, state= parameter. + +############################################################################### + +use warnings; +use strict; + +use Test::More; + +BEGIN { use FindBin; chdir($FindBin::Bin); } + +use lib 'lib'; +use Test::Nginx; +use Test::Nginx::Stream qw/ stream /; + +############################################################################### + +select STDERR; $| = 1; +select STDOUT; $| = 1; + +my $t = Test::Nginx->new()->has(qw/stream/) + ->write_file_expand('nginx.conf', <<'EOF'); + +%%TEST_GLOBALS%% + +daemon off; + +events { +} + +stream { + %%TEST_GLOBALS_STREAM%% + + js_import test.js; + + js_shared_dict_zone zone=foo:32k state=foo.json; + + server { + listen 127.0.0.1:8081; + js_preread test.preread_verify; + proxy_pass 127.0.0.1:8090; + } +} + +EOF + +$t->write_file('foo.json', <<EOF); +{"QZ":{"value":"1"},"QQ":{"value":"1"}} +EOF + +$t->write_file('test.js', <<EOF); + function preread_verify(s) { + var collect = Buffer.from([]); + + s.on('upstream', function (data, flags) { + collect = Buffer.concat([collect, data]); + + if (collect.length >= 4 && collect.readUInt16BE(0) == 0xabcd) { + let id = collect.slice(2,4); + + ngx.shared.foo.get(id) ? s.done(): s.deny(); + + } else if (collect.length) { + s.deny(); + } + }); + } + + export default { preread_verify }; + +EOF + +$t->try_run('no js_shared_dict_zone with state='); + +$t->plan(2); + +$t->run_daemon(\&stream_daemon, port(8090)); +$t->waitforsocket('127.0.0.1:' . port(8090)); + +############################################################################### + +is(stream('127.0.0.1:' . port(8081))->io("\xAB\xCDQY##"), "", + 'access failed, QY is not in the shared dict'); +is(stream('127.0.0.1:' . port(8081))->io("\xAB\xCDQZ##"), "\xAB\xCDQZ##", + 'access granted'); + +############################################################################### + +sub stream_daemon { + my $server = IO::Socket::INET->new( + Proto => 'tcp', + LocalAddr => '127.0.0.1:' . port(8090), + Listen => 5, + Reuse => 1 + ) + or die "Can't create listening socket: $!\n"; + + local $SIG{PIPE} = 'IGNORE'; + + while (my $client = $server->accept()) { + $client->autoflush(1); + + log2c("(new connection $client)"); + + $client->sysread(my $buffer, 65536) or next; + + log2i("$client $buffer"); + + log2o("$client $buffer"); + + $client->syswrite($buffer); + + close $client; + } +} + +sub log2i { Test::Nginx::log_core('|| <<', @_); } +sub log2o { Test::Nginx::log_core('|| >>', @_); } +sub log2c { Test::Nginx::log_core('||', @_); } + +sub get { + my ($url, %extra) = @_; + + my $s = IO::Socket::INET->new( + Proto => 'tcp', + PeerAddr => '127.0.0.1:' . port(8082) + ) or die "Can't connect to nginx: $!\n"; + + return http_get($url, socket => $s); +} + +############################################################################### |