Code coverage testing with gcov. Documentation is in the regression test

chapter.

Author: Michelle Caisse <Michelle.Caisse@Sun.COM>

7 files changed, 361 insertions, 15 deletions

diff --git a/GNUmakefile.in b/GNUmakefile.in
index c71f8a60883..4335c12b002 100644
--- a/GNUmakefile.in
+++ b/GNUmakefile.in
@@ -1,7 +1,7 @@
 #
 # PostgreSQL top level makefile
 #
-# $PostgreSQL: pgsql/GNUmakefile.in,v 1.47 2008/03/18 16:24:50 petere Exp $
+# $PostgreSQL: pgsql/GNUmakefile.in,v 1.48 2008/09/05 12:11:17 petere Exp $
 #
 
 subdir =
@@ -63,6 +63,25 @@ GNUmakefile: GNUmakefile.in $(top_builddir)/config.status
 
 ##########################################################################
 
+coverage:
+	$(MAKE) -C src/backend $@
+
+.PHONY: coverage-html
+coverage-html: coverage
+	rm -rf coverage
+	mkdir coverage
+	$(GENHTML) --show-details --legend --output-directory=coverage --title=PostgreSQL --num-spaces=4 `find src/backend -name lcov.info -print`
+
+ifeq ($(enable_coverage),yes)
+clean distclean maintainer-clean: clean-coverage-local
+.PHONY: clean-coverage-local
+clean-coverage-local:
+	rm -rf coverage
+endif
+
+
+##########################################################################
+
 distdir	= postgresql-$(VERSION)
 dummy	= =install=
 garbage = =*  "#"*  ."#"*  *~*  *.orig  *.rej  core  postgresql-*
diff --git a/configure b/configure
index 96a7e1c3d4b..6f87fba2680 100755
--- a/configure
+++ b/configure
@@ -672,6 +672,10 @@ enable_shared
 enable_rpath
 enable_debug
 enable_profiling
+GCOV
+LCOV
+GENHTML
+enable_coverage
 DTRACE
 DTRACEFLAGS
 enable_dtrace
@@ -1356,6 +1360,7 @@ Optional Features:
   --disable-spinlocks     do not use spinlocks
   --enable-debug          build with debugging symbols (-g)
   --enable-profiling      build with profiling enabled
+  --enable-coverage       build with coverage testing instrumentation
   --enable-dtrace         build with DTrace support
   --enable-depend         turn on automatic dependency tracking
   --enable-cassert        enable assertion checks (for debugging)
@@ -2469,6 +2474,178 @@ fi
 
 
 #
+# --enable-coverage enables generation of code coverage metrics with gcov
+#
+
+pgac_args="$pgac_args enable_coverage"
+
+# Check whether --enable-coverage was given.
+if test "${enable_coverage+set}" = set; then
+  enableval=$enable_coverage;
+  case $enableval in
+    yes)
+      :
+      ;;
+    no)
+      :
+      ;;
+    *)
+      { { echo "$as_me:$LINENO: error: no argument expected for --enable-coverage option" >&5
+echo "$as_me: error: no argument expected for --enable-coverage option" >&2;}
+   { (exit 1); exit 1; }; }
+      ;;
+  esac
+
+else
+  enable_coverage=no
+
+fi
+
+
+for ac_prog in gcov
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+{ echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6; }
+if test "${ac_cv_prog_GCOV+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$GCOV"; then
+  ac_cv_prog_GCOV="$GCOV" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+    ac_cv_prog_GCOV="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+IFS=$as_save_IFS
+
+fi
+fi
+GCOV=$ac_cv_prog_GCOV
+if test -n "$GCOV"; then
+  { echo "$as_me:$LINENO: result: $GCOV" >&5
+echo "${ECHO_T}$GCOV" >&6; }
+else
+  { echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6; }
+fi
+
+
+  test -n "$GCOV" && break
+done
+
+if test -z "$GCOV"; then
+  { { echo "$as_me:$LINENO: error: gcov not found" >&5
+echo "$as_me: error: gcov not found" >&2;}
+   { (exit 1); exit 1; }; }
+fi
+for ac_prog in lcov
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+{ echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6; }
+if test "${ac_cv_prog_LCOV+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$LCOV"; then
+  ac_cv_prog_LCOV="$LCOV" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+    ac_cv_prog_LCOV="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+IFS=$as_save_IFS
+
+fi
+fi
+LCOV=$ac_cv_prog_LCOV
+if test -n "$LCOV"; then
+  { echo "$as_me:$LINENO: result: $LCOV" >&5
+echo "${ECHO_T}$LCOV" >&6; }
+else
+  { echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6; }
+fi
+
+
+  test -n "$LCOV" && break
+done
+
+if test -z "$LCOV"; then
+  { { echo "$as_me:$LINENO: error: lcov not found" >&5
+echo "$as_me: error: lcov not found" >&2;}
+   { (exit 1); exit 1; }; }
+fi
+for ac_prog in genhtml
+do
+  # Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+{ echo "$as_me:$LINENO: checking for $ac_word" >&5
+echo $ECHO_N "checking for $ac_word... $ECHO_C" >&6; }
+if test "${ac_cv_prog_GENHTML+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  if test -n "$GENHTML"; then
+  ac_cv_prog_GENHTML="$GENHTML" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+  for ac_exec_ext in '' $ac_executable_extensions; do
+  if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+    ac_cv_prog_GENHTML="$ac_prog"
+    echo "$as_me:$LINENO: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+done
+IFS=$as_save_IFS
+
+fi
+fi
+GENHTML=$ac_cv_prog_GENHTML
+if test -n "$GENHTML"; then
+  { echo "$as_me:$LINENO: result: $GENHTML" >&5
+echo "${ECHO_T}$GENHTML" >&6; }
+else
+  { echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6; }
+fi
+
+
+  test -n "$GENHTML" && break
+done
+
+if test -z "$GENHTML"; then
+  { { echo "$as_me:$LINENO: error: genhtml not found" >&5
+echo "$as_me: error: genhtml not found" >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+
+#
 # DTrace
 #
 
@@ -3580,13 +3757,16 @@ unset CFLAGS
 # CFLAGS are selected so:
 # If the user specifies something in the environment, that is used.
 # else:  If the template file set something, that is used.
+# else:  If coverage was enabled, don't set anything.
 # else:  If the compiler is GCC, then we use -O2.
-# else:  If the compiler is something else, then we use -0.
+# else:  If the compiler is something else, then we use -O.
 
 if test "$ac_env_CFLAGS_set" = set; then
   CFLAGS=$ac_env_CFLAGS_value
 elif test "${CFLAGS+set}" = set; then
   : # (keep what template set)
+elif test "$enable_coverage" = yes; then
+  : # no optimization by default
 elif test "$GCC" = yes; then
   CFLAGS="-O2"
 else
@@ -3961,6 +4141,17 @@ if test "$enable_debug" = yes && test "$ac_cv_prog_cc_g" = yes; then
   CFLAGS="$CFLAGS -g"
 fi
 
+# enable code coverage if --enable-coverage
+if test "$enable_coverage" = yes; then
+  if test "$GCC" = yes; then
+    CFLAGS="$CFLAGS -fprofile-arcs -ftest-coverage"
+  else
+    { { echo "$as_me:$LINENO: error: --enable-coverage is supported only when using GCC" >&5
+echo "$as_me: error: --enable-coverage is supported only when using GCC" >&2;}
+   { (exit 1); exit 1; }; }
+  fi
+fi
+
 # enable profiling if --enable-profiling
 if test "$enable_profiling" = yes && test "$ac_cv_prog_cc_g" = yes; then
   if test "$GCC" = yes; then
@@ -26510,6 +26701,10 @@ enable_shared!$enable_shared$ac_delim
 enable_rpath!$enable_rpath$ac_delim
 enable_debug!$enable_debug$ac_delim
 enable_profiling!$enable_profiling$ac_delim
+GCOV!$GCOV$ac_delim
+LCOV!$LCOV$ac_delim
+GENHTML!$GENHTML$ac_delim
+enable_coverage!$enable_coverage$ac_delim
 DTRACE!$DTRACE$ac_delim
 DTRACEFLAGS!$DTRACEFLAGS$ac_delim
 enable_dtrace!$enable_dtrace$ac_delim
@@ -26549,10 +26744,6 @@ LDFLAGS_SL!$LDFLAGS_SL$ac_delim
 LD!$LD$ac_delim
 with_gnu_ld!$with_gnu_ld$ac_delim
 ld_R_works!$ld_R_works$ac_delim
-RANLIB!$RANLIB$ac_delim
-STRIP!$STRIP$ac_delim
-STRIP_STATIC_LIB!$STRIP_STATIC_LIB$ac_delim
-STRIP_SHARED_LIB!$STRIP_SHARED_LIB$ac_delim
 _ACEOF
 
   if test `sed -n "s/.*$ac_delim\$/X/p" conf$$subs.sed | grep -c X` = 97; then
@@ -26594,6 +26785,10 @@ _ACEOF
 ac_delim='%!_!# '
 for ac_last_try in false false false false false :; do
   cat >conf$$subs.sed <<_ACEOF
+RANLIB!$RANLIB$ac_delim
+STRIP!$STRIP$ac_delim
+STRIP_STATIC_LIB!$STRIP_STATIC_LIB$ac_delim
+STRIP_SHARED_LIB!$STRIP_SHARED_LIB$ac_delim
 TAR!$TAR$ac_delim
 LN_S!$LN_S$ac_delim
 AWK!$AWK$ac_delim
@@ -26644,7 +26839,7 @@ vpath_build!$vpath_build$ac_delim
 LTLIBOBJS!$LTLIBOBJS$ac_delim
 _ACEOF
 
-  if test `sed -n "s/.*$ac_delim\$/X/p" conf$$subs.sed | grep -c X` = 48; then
+  if test `sed -n "s/.*$ac_delim\$/X/p" conf$$subs.sed | grep -c X` = 52; then
     break
   elif $ac_last_try; then
     { { echo "$as_me:$LINENO: error: could not make $CONFIG_STATUS" >&5
diff --git a/configure.in b/configure.in
index e351bd1c85a..97c220c5c18 100644
--- a/configure.in
+++ b/configure.in
@@ -1,5 +1,5 @@
 dnl Process this file with autoconf to produce a configure script.
-dnl $PostgreSQL: pgsql/configure.in,v 1.565 2008/08/29 13:02:32 petere Exp $
+dnl $PostgreSQL: pgsql/configure.in,v 1.566 2008/09/05 12:11:18 petere Exp $
 dnl
 dnl Developers, please strive to achieve this order:
 dnl
@@ -204,6 +204,25 @@ PGAC_ARG_BOOL(enable, profiling, no,
 AC_SUBST(enable_profiling)
 
 #
+# --enable-coverage enables generation of code coverage metrics with gcov
+#
+PGAC_ARG_BOOL(enable, coverage, no,
+              [  --enable-coverage       build with coverage testing instrumentation])
+AC_CHECK_PROGS(GCOV, gcov)
+if test -z "$GCOV"; then
+  AC_MSG_ERROR([gcov not found])
+fi
+AC_CHECK_PROGS(LCOV, lcov)
+if test -z "$LCOV"; then
+  AC_MSG_ERROR([lcov not found])
+fi
+AC_CHECK_PROGS(GENHTML, genhtml)
+if test -z "$GENHTML"; then
+  AC_MSG_ERROR([genhtml not found])
+fi
+AC_SUBST(enable_coverage)
+
+#
 # DTrace
 #
 PGAC_ARG_BOOL(enable, dtrace, no,
@@ -370,13 +389,16 @@ unset CFLAGS
 # CFLAGS are selected so:
 # If the user specifies something in the environment, that is used.
 # else:  If the template file set something, that is used.
+# else:  If coverage was enabled, don't set anything.
 # else:  If the compiler is GCC, then we use -O2.
-# else:  If the compiler is something else, then we use -0.
+# else:  If the compiler is something else, then we use -O.
 
 if test "$ac_env_CFLAGS_set" = set; then
   CFLAGS=$ac_env_CFLAGS_value
 elif test "${CFLAGS+set}" = set; then
   : # (keep what template set)
+elif test "$enable_coverage" = yes; then
+  : # no optimization by default
 elif test "$GCC" = yes; then
   CFLAGS="-O2"
 else
@@ -415,6 +437,15 @@ if test "$enable_debug" = yes && test "$ac_cv_prog_cc_g" = yes; then
   CFLAGS="$CFLAGS -g"
 fi
 
+# enable code coverage if --enable-coverage
+if test "$enable_coverage" = yes; then
+  if test "$GCC" = yes; then
+    CFLAGS="$CFLAGS -fprofile-arcs -ftest-coverage"
+  else
+    AC_MSG_ERROR([--enable-coverage is supported only when using GCC])
+  fi
+fi
+
 # enable profiling if --enable-profiling
 if test "$enable_profiling" = yes && test "$ac_cv_prog_cc_g" = yes; then
   if test "$GCC" = yes; then
diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml
index a5dfa8d0810..cee4681ce39 100644
--- a/doc/src/sgml/installation.sgml
+++ b/doc/src/sgml/installation.sgml
@@ -1,4 +1,4 @@
-<!-- $PostgreSQL: pgsql/doc/src/sgml/installation.sgml,v 1.312 2008/08/29 13:02:32 petere Exp $ -->
+<!-- $PostgreSQL: pgsql/doc/src/sgml/installation.sgml,v 1.313 2008/09/05 12:11:18 petere Exp $ -->
 
 <chapter id="installation">
  <title><![%standalone-include[<productname>PostgreSQL</>]]>
@@ -1234,6 +1234,21 @@ su - postgres
       </varlistentry>
 
       <varlistentry>
+       <term><option>--enable-coverage</option></term>
+       <listitem>
+        <para>
+         If using GCC, all programs and libraries are compiled with
+         code coverage testing instrumentation.  When run, they
+         generate files in the build directory with code coverage
+         metrics.
+         <![%standalone-ignore[See <xref linkend="regress-coverage">
+         for more information.]]> This option is for use only with GCC
+         and when doing development work.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry>
        <term><option>--enable-profiling</option></term>
        <listitem>
         <para>
diff --git a/doc/src/sgml/regress.sgml b/doc/src/sgml/regress.sgml
index d09137fcc0f..0e082603bf1 100644
--- a/doc/src/sgml/regress.sgml
+++ b/doc/src/sgml/regress.sgml
@@ -1,4 +1,4 @@
-<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.59 2008/05/30 00:04:32 tgl Exp $ -->
+<!-- $PostgreSQL: pgsql/doc/src/sgml/regress.sgml,v 1.60 2008/09/05 12:11:18 petere Exp $ -->
 
  <chapter id="regress">
   <title id="regress-title">Regression Tests</title>
@@ -419,5 +419,37 @@ float8:out:i.86-.*-openbsd=float8-small-is-zero.out
    </para>
     
   </sect1>
-  
+
+  <sect1 id="regress-coverage">
+   <title>Test Coverage Examination</title>
+
+   <para>
+    The PostgreSQL source code can be compiled with coverage testing
+    instrumentation, so that it becomes possible to examine which
+    parts of the code are covered by the regression tests or any other
+    test suite that is run with the code.  This is currently supported
+    when compiling with GCC and requires the <command>gcov</command>
+    and <command>lcov</command> programs.
+   </para>
+
+   <para>
+    A typical workflow would look like this:
+<screen>
+./configure --enable-coverage ... OTHER OPTIONS ...
+gmake
+gmake check # or other test suite
+gmake coverage-html
+</screen>
+    Then point your HTML browser
+    to <filename>coverage/index.html</filename>.
+   </para>
+
+   <para>
+    To reset the execution counts between test runs, run
+<screen>
+gmake coverage-clean
+</screen>
+   </para>
+  </sect1>
+
 </chapter>
diff --git a/src/Makefile.global.in b/src/Makefile.global.in
index 6212a1467ff..def667a1039 100644
--- a/src/Makefile.global.in
+++ b/src/Makefile.global.in
@@ -1,5 +1,5 @@
 # -*-makefile-*-
-# $PostgreSQL: pgsql/src/Makefile.global.in,v 1.242 2008/08/29 13:02:32 petere Exp $
+# $PostgreSQL: pgsql/src/Makefile.global.in,v 1.243 2008/09/05 12:11:18 petere Exp $
 
 #------------------------------------------------------------------------------
 # All PostgreSQL makefiles include this file and use the variables it sets,
@@ -18,7 +18,7 @@
 #
 # Meta configuration
 
-.PHONY: all install install-strip installdirs uninstall clean distclean maintainer-clean distprep check installcheck maintainer-check
+.PHONY: all install install-strip installdirs uninstall clean distclean maintainer-clean distprep check installcheck maintainer-check coverage
 .SILENT: installdirs
 
 # make `all' the default target
@@ -165,6 +165,7 @@ enable_rpath	= @enable_rpath@
 enable_nls	= @enable_nls@
 enable_debug	= @enable_debug@
 enable_dtrace	= @enable_dtrace@
+enable_coverage	= @enable_coverage@
 enable_thread_safety	= @enable_thread_safety@
 
 python_includespec	= @python_includespec@
@@ -291,6 +292,17 @@ JADE	= @JADE@
 NSGMLS	= @NSGMLS@
 SGMLSPL	= @SGMLSPL@
 
+# Code coverage
+
+GCOV = @GCOV@
+LCOV = @LCOV@
+GENHTML = @GENHTML@
+
+ifeq ($(enable_coverage),yes)
+# ccache loses .gcno files
+export CCACHE_DISABLE = 1
+endif
+
 # Feature settings
 
 DEF_PGPORT = @default_port@
@@ -574,3 +586,39 @@ include $(top_srcdir)/src/nls-global.mk
 
 endif # nls.mk
 endif # enable_nls
+
+
+##########################################################################
+#
+# Coverage
+
+ifeq ($(enable_coverage), yes)
+
+# There is a strange interaction between lcov and existing .gcov
+# output files.  Hence the rm command and the ordering dependency.
+
+gcda_files := $(wildcard *.gcda)
+
+lcov.info:
+	rm -f *.gcov
+ifneq (,$(gcda_files))
+	$(LCOV) -d . -c -o $@ $(LCOVFLAGS)
+endif
+
+%.c.gcov: %.gcda | lcov.info
+	$(GCOV) -b -f -p -o . $(GCOVFLAGS) $*.c >$*.c.gcov.out
+
+
+# hook for clean-up
+clean distclean maintainer-clean: clean-coverage
+
+.PHONY: clean-coverage
+clean-coverage:
+	rm -f *.gcda *.gcno lcov.info *.gcov *.gcov.out
+
+
+# User-callable target to reset counts between test runs
+coverage-clean:
+	rm -f `find . -name '*.gcda' -print`
+
+endif # enable_coverage
diff --git a/src/backend/common.mk b/src/backend/common.mk
index 82a9ab6fc10..c617c672765 100644
--- a/src/backend/common.mk
+++ b/src/backend/common.mk
@@ -1,7 +1,7 @@
 #
 # Common make rules for backend
 #
-# $PostgreSQL: pgsql/src/backend/common.mk,v 1.7 2008/03/17 18:24:56 petere Exp $
+# $PostgreSQL: pgsql/src/backend/common.mk,v 1.8 2008/09/05 12:11:18 petere Exp $
 #
 
 # When including this file, set OBJS to the object files created in
@@ -46,3 +46,9 @@ ifdef SUBDIRS
 	for dir in $(SUBDIRS); do $(MAKE) -C $$dir clean || exit; done
 endif
 	rm -f $(subsysfilename) $(OBJS)
+
+
+coverage: $(gcda_files:.gcda=.c.gcov) lcov.info
+ifdef SUBDIRS
+	for dir in $(SUBDIRS); do $(MAKE) -C $$dir coverage || exit; done
+endif