Change function call information to be variable length.

Before this change FunctionCallInfoData, the struct arguments etc for
V1 function calls are stored in, always had space for
FUNC_MAX_ARGS/100 arguments, storing datums and their nullness in two
arrays.  For nearly every function call 100 arguments is far more than
needed, therefore wasting memory. Arg and argnull being two separate
arrays also guarantees that to access a single argument, two
cachelines have to be touched.

Change the layout so there's a single variable-length array with pairs
of value / isnull. That drastically reduces memory consumption for
most function calls (on x86-64 a two argument function now uses
64bytes, previously 936 bytes), and makes it very likely that argument
value and its nullness are on the same cacheline.

Arguments are stored in a new NullableDatum struct, which, due to
padding, needs more memory per argument than before. But as usually
far fewer arguments are stored, and individual arguments are cheaper
to access, that's still a clear win.  It's likely that there's other
places where conversion to NullableDatum arrays would make sense,
e.g. TupleTableSlots, but that's for another commit.

Because the function call information is now variable-length
allocations have to take the number of arguments into account. For
heap allocations that can be done with SizeForFunctionCallInfoData(),
for on-stack allocations there's a new LOCAL_FCINFO(name, nargs) macro
that helps to allocate an appropriately sized and aligned variable.

Some places with stack allocation function call information don't know
the number of arguments at compile time, and currently variably sized
stack allocations aren't allowed in postgres. Therefore allow for
FUNC_MAX_ARGS space in these cases. They're not that common, so for
now that seems acceptable.

Because of the need to allocate FunctionCallInfo of the appropriate
size, older extensions may need to update their code. To avoid subtle
breakages, the FunctionCallInfoData struct has been renamed to
FunctionCallInfoBaseData. Most code only references FunctionCallInfo,
so that shouldn't cause much collateral damage.

This change is also a prerequisite for more efficient expression JIT
compilation (by allocating the function call information on the stack,
allowing LLVM to optimize it away); previously the size of the call
information caused problems inside LLVM's optimizer.

Author: Andres Freund
Reviewed-By: Tom Lane
Discussion: https://postgr.es/m/20180605172952.x34m5uz6ju6enaem@alap3.anarazel.de

1 files changed, 39 insertions, 107 deletions

diff --git a/src/backend/jit/llvm/llvmjit_expr.c b/src/backend/jit/llvm/llvmjit_expr.c
index 70b55dd202e..351f6ec7e1a 100644
--- a/src/backend/jit/llvm/llvmjit_expr.c
+++ b/src/backend/jit/llvm/llvmjit_expr.c
@@ -570,7 +570,6 @@ llvm_compile_expr(ExprState *state)
 					LLVMBasicBlockRef b_nonull;
 					int			argno;
 					LLVMValueRef v_fcinfo;
-					LLVMValueRef v_argnullp;
 					LLVMBasicBlockRef *b_checkargnulls;
 
 					/*
@@ -587,12 +586,6 @@ llvm_compile_expr(ExprState *state)
 					v_fcinfo =
 						l_ptr_const(fcinfo, l_ptr(StructFunctionCallInfoData));
 
-					v_argnullp =
-						LLVMBuildStructGEP(b,
-										   v_fcinfo,
-										   FIELDNO_FUNCTIONCALLINFODATA_ARGNULL,
-										   "v_argnullp");
-
 					/*
 					 * set resnull to true, if the function is actually
 					 * called, it'll be reset
@@ -624,8 +617,7 @@ llvm_compile_expr(ExprState *state)
 							b_argnotnull = b_checkargnulls[argno + 1];
 
 						/* and finally load & check NULLness of arg */
-						v_argisnull = l_load_struct_gep(b, v_argnullp,
-														argno, "");
+						v_argisnull = l_funcnull(b, v_fcinfo, argno);
 						LLVMBuildCondBr(b,
 										LLVMBuildICmp(b, LLVMIntEQ,
 													  v_argisnull,
@@ -1276,10 +1268,6 @@ llvm_compile_expr(ExprState *state)
 					LLVMValueRef v_fn_addr_out,
 								v_fn_addr_in;
 					LLVMValueRef v_fcinfo_in_isnullp;
-					LLVMValueRef v_in_argp,
-								v_out_argp;
-					LLVMValueRef v_in_argnullp,
-								v_out_argnullp;
 					LLVMValueRef v_retval;
 					LLVMValueRef v_resvalue;
 					LLVMValueRef v_resnull;
@@ -1313,22 +1301,6 @@ llvm_compile_expr(ExprState *state)
 						LLVMBuildStructGEP(b, v_fcinfo_in,
 										   FIELDNO_FUNCTIONCALLINFODATA_ISNULL,
 										   "v_fcinfo_in_isnull");
-					v_out_argnullp =
-						LLVMBuildStructGEP(b, v_fcinfo_out,
-										   FIELDNO_FUNCTIONCALLINFODATA_ARGNULL,
-										   "v_fcinfo_out_argnullp");
-					v_in_argnullp =
-						LLVMBuildStructGEP(b, v_fcinfo_in,
-										   FIELDNO_FUNCTIONCALLINFODATA_ARGNULL,
-										   "v_fcinfo_in_argnullp");
-					v_out_argp =
-						LLVMBuildStructGEP(b, v_fcinfo_out,
-										   FIELDNO_FUNCTIONCALLINFODATA_ARG,
-										   "v_fcinfo_out_argp");
-					v_in_argp =
-						LLVMBuildStructGEP(b, v_fcinfo_in,
-										   FIELDNO_FUNCTIONCALLINFODATA_ARG,
-										   "v_fcinfo_in_argp");
 
 					/* output functions are not called on nulls */
 					v_resnull = LLVMBuildLoad(b, v_resnullp, "");
@@ -1348,11 +1320,10 @@ llvm_compile_expr(ExprState *state)
 					/* set arg[0] */
 					LLVMBuildStore(b,
 								   v_resvalue,
-								   LLVMBuildStructGEP(b, v_out_argp, 0, ""));
+								   l_funcvaluep(b, v_fcinfo_out, 0));
 					LLVMBuildStore(b,
 								   l_sbool_const(0),
-								   LLVMBuildStructGEP(b, v_out_argnullp,
-													  0, ""));
+								   l_funcnullp(b, v_fcinfo_out, 0));
 					/* and call output function (can never return NULL) */
 					v_output = LLVMBuildCall(b, v_fn_addr_out, &v_fcinfo_out,
 											 1, "funccall_coerce_out");
@@ -1399,9 +1370,9 @@ llvm_compile_expr(ExprState *state)
 					/* set arguments */
 					/* arg0: output */
 					LLVMBuildStore(b, v_output,
-								   LLVMBuildStructGEP(b, v_in_argp, 0, ""));
+								   l_funcvaluep(b, v_fcinfo_in, 0));
 					LLVMBuildStore(b, v_resnull,
-								   LLVMBuildStructGEP(b, v_in_argnullp, 0, ""));
+								   l_funcnullp(b, v_fcinfo_in, 0));
 
 					/* arg1: ioparam: preset in execExpr.c */
 					/* arg2: typmod: preset in execExpr.c  */
@@ -1426,7 +1397,6 @@ llvm_compile_expr(ExprState *state)
 					LLVMValueRef v_fcinfo;
 					LLVMValueRef v_fcinfo_isnull;
 
-					LLVMValueRef v_argnullp;
 					LLVMValueRef v_argnull0,
 								v_argisnull0;
 					LLVMValueRef v_argnull1,
@@ -1449,18 +1419,11 @@ llvm_compile_expr(ExprState *state)
 
 					v_fcinfo = l_ptr_const(fcinfo, l_ptr(StructFunctionCallInfoData));
 
-					v_argnullp =
-						LLVMBuildStructGEP(b,
-										   v_fcinfo,
-										   FIELDNO_FUNCTIONCALLINFODATA_ARGNULL,
-										   "v_argnullp");
-
-					/* load argnull[0|1] for both arguments */
-					v_argnull0 = l_load_struct_gep(b, v_argnullp, 0, "");
+					/* load args[0|1].isnull for both arguments */
+					v_argnull0 = l_funcnull(b, v_fcinfo, 0);
 					v_argisnull0 = LLVMBuildICmp(b, LLVMIntEQ, v_argnull0,
 												 l_sbool_const(1), "");
-
-					v_argnull1 = l_load_struct_gep(b, v_argnullp, 1, "");
+					v_argnull1 = l_funcnull(b, v_fcinfo, 1);
 					v_argisnull1 = LLVMBuildICmp(b, LLVMIntEQ, v_argnull1,
 												 l_sbool_const(1), "");
 
@@ -1532,11 +1495,9 @@ llvm_compile_expr(ExprState *state)
 
 					LLVMValueRef v_fcinfo;
 					LLVMValueRef v_fcinfo_isnull;
-					LLVMValueRef v_argnullp;
 					LLVMValueRef v_argnull0;
 					LLVMValueRef v_argnull1;
 					LLVMValueRef v_anyargisnull;
-					LLVMValueRef v_argp;
 					LLVMValueRef v_arg0;
 					LLVMBasicBlockRef b_hasnull;
 					LLVMBasicBlockRef b_nonull;
@@ -1553,21 +1514,9 @@ llvm_compile_expr(ExprState *state)
 
 					v_fcinfo = l_ptr_const(fcinfo, l_ptr(StructFunctionCallInfoData));
 
-					v_argnullp =
-						LLVMBuildStructGEP(b,
-										   v_fcinfo,
-										   FIELDNO_FUNCTIONCALLINFODATA_ARGNULL,
-										   "v_argnullp");
-
-					v_argp =
-						LLVMBuildStructGEP(b,
-										   v_fcinfo,
-										   FIELDNO_FUNCTIONCALLINFODATA_ARG,
-										   "v_argp");
-
 					/* if either argument is NULL they can't be equal */
-					v_argnull0 = l_load_struct_gep(b, v_argnullp, 0, "");
-					v_argnull1 = l_load_struct_gep(b, v_argnullp, 1, "");
+					v_argnull0 = l_funcnull(b, v_fcinfo, 0);
+					v_argnull1 = l_funcnull(b, v_fcinfo, 1);
 
 					v_anyargisnull =
 						LLVMBuildOr(b,
@@ -1581,7 +1530,7 @@ llvm_compile_expr(ExprState *state)
 
 					/* one (or both) of the arguments are null, return arg[0] */
 					LLVMPositionBuilderAtEnd(b, b_hasnull);
-					v_arg0 = l_load_struct_gep(b, v_argp, 0, "");
+					v_arg0 = l_funcvalue(b, v_fcinfo, 0);
 					LLVMBuildStore(b, v_argnull0, v_resnullp);
 					LLVMBuildStore(b, v_arg0, v_resvaluep);
 					LLVMBuildBr(b, opblocks[i + 1]);
@@ -1680,7 +1629,6 @@ llvm_compile_expr(ExprState *state)
 					if (op->d.rowcompare_step.finfo->fn_strict)
 					{
 						LLVMValueRef v_fcinfo;
-						LLVMValueRef v_argnullp;
 						LLVMValueRef v_argnull0;
 						LLVMValueRef v_argnull1;
 						LLVMValueRef v_anyargisnull;
@@ -1688,13 +1636,8 @@ llvm_compile_expr(ExprState *state)
 						v_fcinfo = l_ptr_const(fcinfo,
 											   l_ptr(StructFunctionCallInfoData));
 
-						v_argnullp =
-							LLVMBuildStructGEP(b, v_fcinfo,
-											   FIELDNO_FUNCTIONCALLINFODATA_ARGNULL,
-											   "v_argnullp");
-
-						v_argnull0 = l_load_struct_gep(b, v_argnullp, 0, "");
-						v_argnull1 = l_load_struct_gep(b, v_argnullp, 1, "");
+						v_argnull0 = l_funcnull(b, v_fcinfo, 0);
+						v_argnull1 = l_funcnull(b, v_fcinfo, 1);
 
 						v_anyargisnull =
 							LLVMBuildOr(b,
@@ -2021,7 +1964,6 @@ llvm_compile_expr(ExprState *state)
 				{
 					FunctionCallInfo fcinfo = op->d.agg_deserialize.fcinfo_data;
 					LLVMValueRef v_fcinfo;
-					LLVMValueRef v_argnullp;
 					LLVMValueRef v_argnull0;
 					LLVMBasicBlockRef b_deserialize;
 
@@ -2030,14 +1972,7 @@ llvm_compile_expr(ExprState *state)
 
 					v_fcinfo = l_ptr_const(fcinfo,
 										   l_ptr(StructFunctionCallInfoData));
-
-					v_argnullp =
-						LLVMBuildStructGEP(b,
-										   v_fcinfo,
-										   FIELDNO_FUNCTIONCALLINFODATA_ARGNULL,
-										   "v_argnullp");
-					v_argnull0 =
-						l_load_struct_gep(b, v_argnullp, 0, "v_argnull0");
+					v_argnull0 = l_funcnull(b, v_fcinfo, 0);
 
 					LLVMBuildCondBr(b,
 									LLVMBuildICmp(b,
@@ -2079,20 +2014,24 @@ llvm_compile_expr(ExprState *state)
 					break;
 				}
 
-			case EEOP_AGG_STRICT_INPUT_CHECK:
+			case EEOP_AGG_STRICT_INPUT_CHECK_NULLS:
+			case EEOP_AGG_STRICT_INPUT_CHECK_ARGS:
 				{
 					int			nargs = op->d.agg_strict_input_check.nargs;
+					NullableDatum *args = op->d.agg_strict_input_check.args;
 					bool	   *nulls = op->d.agg_strict_input_check.nulls;
 					int			jumpnull;
 					int			argno;
 
-					LLVMValueRef v_nullp;
+					LLVMValueRef v_argsp;
+					LLVMValueRef v_nullsp;
 					LLVMBasicBlockRef *b_checknulls;
 
 					Assert(nargs > 0);
 
 					jumpnull = op->d.agg_strict_input_check.jumpnull;
-					v_nullp = l_ptr_const(nulls, l_ptr(TypeStorageBool));
+					v_argsp = l_ptr_const(args, l_ptr(StructNullableDatum));
+					v_nullsp = l_ptr_const(nulls, l_ptr(TypeStorageBool));
 
 					/* create blocks for checking args */
 					b_checknulls = palloc(sizeof(LLVMBasicBlockRef *) * nargs);
@@ -2120,7 +2059,18 @@ llvm_compile_expr(ExprState *state)
 						else
 							b_argnotnull = b_checknulls[argno + 1];
 
-						v_argisnull = l_load_gep1(b, v_nullp, v_argno, "");
+						if (opcode == EEOP_AGG_STRICT_INPUT_CHECK_NULLS)
+							v_argisnull = l_load_gep1(b, v_nullsp, v_argno, "");
+						else
+						{
+							LLVMValueRef v_argn;
+
+							v_argn = LLVMBuildGEP(b, v_argsp, &v_argno, 1, "");
+							v_argisnull =
+								l_load_struct_gep(b, v_argn,
+												  FIELDNO_NULLABLE_DATUM_ISNULL,
+												  "");
+						}
 
 						LLVMBuildCondBr(b,
 										LLVMBuildICmp(b,
@@ -2291,8 +2241,6 @@ llvm_compile_expr(ExprState *state)
 					LLVMValueRef v_aggstatep;
 					LLVMValueRef v_fcinfo;
 					LLVMValueRef v_fcinfo_isnull;
-					LLVMValueRef v_argp,
-								v_argnullp;
 
 					LLVMValueRef v_transvaluep;
 					LLVMValueRef v_transnullp;
@@ -2319,7 +2267,7 @@ llvm_compile_expr(ExprState *state)
 					aggstate = op->d.agg_trans.aggstate;
 					pertrans = op->d.agg_trans.pertrans;
 
-					fcinfo = &pertrans->transfn_fcinfo;
+					fcinfo = pertrans->transfn_fcinfo;
 
 					v_aggstatep = l_ptr_const(aggstate,
 											  l_ptr(StructAggState));
@@ -2344,18 +2292,6 @@ llvm_compile_expr(ExprState *state)
 
 					v_fcinfo = l_ptr_const(fcinfo,
 										   l_ptr(StructFunctionCallInfoData));
-
-					v_argnullp =
-						LLVMBuildStructGEP(b,
-										   v_fcinfo,
-										   FIELDNO_FUNCTIONCALLINFODATA_ARGNULL,
-										   "v_argnullp");
-					v_argp =
-						LLVMBuildStructGEP(b,
-										   v_fcinfo,
-										   FIELDNO_FUNCTIONCALLINFODATA_ARG,
-										   "v_argp");
-
 					v_aggcontext = l_ptr_const(op->d.agg_trans.aggcontext,
 											   l_ptr(StructExprContext));
 
@@ -2387,7 +2323,7 @@ llvm_compile_expr(ExprState *state)
 									l_ptr(StructMemoryContextData));
 					v_oldcontext = l_mcxt_switch(mod, b, v_tmpcontext);
 
-					/* store transvalue in fcinfo->arg/argnull[0] */
+					/* store transvalue in fcinfo->args[0] */
 					v_transvaluep =
 						LLVMBuildStructGEP(b, v_pergroupp,
 										   FIELDNO_AGGSTATEPERGROUPDATA_TRANSVALUE,
@@ -2399,10 +2335,10 @@ llvm_compile_expr(ExprState *state)
 					LLVMBuildStore(b,
 								   LLVMBuildLoad(b, v_transvaluep,
 												 "transvalue"),
-								   LLVMBuildStructGEP(b, v_argp, 0, ""));
+								   l_funcvaluep(b, v_fcinfo, 0));
 					LLVMBuildStore(b,
 								   LLVMBuildLoad(b, v_transnullp, "transnull"),
-								   LLVMBuildStructGEP(b, v_argnullp, 0, ""));
+								   l_funcnullp(b, v_fcinfo, 0));
 
 					/* and invoke transition function */
 					v_retval = BuildV1Call(context, b, mod, fcinfo,
@@ -2589,12 +2525,8 @@ BuildV1Call(LLVMJitContext *context, LLVMBuilderRef b,
 		LLVMValueRef v_lifetime = create_LifetimeEnd(mod);
 		LLVMValueRef params[2];
 
-		params[0] = l_int64_const(sizeof(fcinfo->arg));
-		params[1] = l_ptr_const(fcinfo->arg, l_ptr(LLVMInt8Type()));
-		LLVMBuildCall(b, v_lifetime, params, lengthof(params), "");
-
-		params[0] = l_int64_const(sizeof(fcinfo->argnull));
-		params[1] = l_ptr_const(fcinfo->argnull, l_ptr(LLVMInt8Type()));
+		params[0] = l_int64_const(sizeof(NullableDatum) * fcinfo->nargs);
+		params[1] = l_ptr_const(fcinfo->args, l_ptr(LLVMInt8Type()));
 		LLVMBuildCall(b, v_lifetime, params, lengthof(params), "");
 
 		params[0] = l_int64_const(sizeof(fcinfo->isnull));