Default to hidden visibility for extension libraries where possible

Until now postgres built extension libraries with global visibility, i.e.
exporting all symbols.  On the one platform where that behavior is not
natively available, namely windows, we emulate it by analyzing the input files
to the shared library and exporting all the symbols therein.

Not exporting all symbols is actually desirable, as it can improve loading
speed, reduces the likelihood of symbol conflicts and can improve intra
extension library function call performance. It also makes the non-windows
builds more similar to windows builds.

Additionally, with meson implementing the export-all-symbols behavior for
windows, turns out to be more verbose than desirable.

This patch adds support for hiding symbols by default and, to counteract that,
explicit symbol visibility annotation for compilers that support
__attribute__((visibility("default"))) and -fvisibility=hidden. That is
expected to be most, if not all, compilers except msvc (for which we already
support explicit symbol export annotations).

Now that extension library symbols are explicitly exported, we don't need to
export all symbols on windows anymore, hence remove that behavior from
src/tools/msvc. The supporting code can't be removed, as we still need to
export all symbols from the main postgres binary.

Author: Andres Freund <andres@anarazel.de>
Author: Tom Lane <tgl@sss.pgh.pa.us>
Discussion: https://postgr.es/m/20211101020311.av6hphdl6xbjbuif@alap3.anarazel.de

7 files changed, 33 insertions, 12 deletions

diff --git a/src/Makefile.global.in b/src/Makefile.global.in
index 138d66ac006..bb177a81622 100644
--- a/src/Makefile.global.in
+++ b/src/Makefile.global.in
@@ -258,6 +258,9 @@ SUN_STUDIO_CC = @SUN_STUDIO_CC@
 CXX = @CXX@
 CFLAGS = @CFLAGS@
 CFLAGS_SL = @CFLAGS_SL@
+# *_MODULE are for flags applied to extension libraries
+CFLAGS_SL_MODULE = @CFLAGS_SL_MODULE@
+CXXFLAGS_SL_MODULE = @CXXFLAGS_SL_MODULE@
 CFLAGS_UNROLL_LOOPS = @CFLAGS_UNROLL_LOOPS@
 CFLAGS_VECTORIZE = @CFLAGS_VECTORIZE@
 CFLAGS_SSE42 = @CFLAGS_SSE42@
diff --git a/src/Makefile.shlib b/src/Makefile.shlib
index 6df96c634b6..2396bc247e5 100644
--- a/src/Makefile.shlib
+++ b/src/Makefile.shlib
@@ -218,6 +218,19 @@ ifeq ($(PORTNAME), win32)
 endif
 
 
+# If the shared library doesn't have an export file, mark all symbols not
+# explicitly exported using PGDLLEXPORT as hidden. We can't pass these flags
+# when building a library with explicit exports, as the symbols would be
+# hidden before the linker script / exported symbol list takes effect.
+#
+# This is duplicated in pgxs.mk for MODULES style libraries.
+ifeq ($(SHLIB_EXPORTS),)
+  # LDFLAGS_SL addition not strictly needed, CFLAGS used everywhere, but ...
+  override LDFLAGS_SL += $(CFLAGS_SL_MODULE)
+  override CFLAGS += $(CFLAGS_SL_MODULE)
+  override CXXFLAGS += $(CXXFLAGS_SL_MODULE)
+endif
+
 
 ##
 ## BUILD
diff --git a/src/include/c.h b/src/include/c.h
index 863a16c6a6c..2cc2784750e 100644
--- a/src/include/c.h
+++ b/src/include/c.h
@@ -1347,14 +1347,19 @@ extern unsigned long long strtoull(const char *str, char **endptr, int base);
 
 /*
  * Use "extern PGDLLEXPORT ..." to declare functions that are defined in
- * loadable modules and need to be callable by the core backend.  (Usually,
- * this is not necessary because our build process automatically exports
- * such symbols, but sometimes manual marking is required.)
- * No special marking is required on most ports.
+ * loadable modules and need to be callable by the core backend or other
+ * loadable modules.
+ * If the compiler knows __attribute__((visibility("*"))), we use that,
+ * unless we already have a platform-specific definition.  Otherwise,
+ * no special marking is required.
  */
 #ifndef PGDLLEXPORT
+#ifdef HAVE_VISIBILITY_ATTRIBUTE
+#define PGDLLEXPORT __attribute__((visibility("default")))
+#else
 #define PGDLLEXPORT
 #endif
+#endif
 
 /*
  * The following is used as the arg list for signal handlers.  Any ports
diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in
index 7133c3dc66b..529fb84a86c 100644
--- a/src/include/pg_config.h.in
+++ b/src/include/pg_config.h.in
@@ -700,6 +700,9 @@
 /* Define to 1 if you have the <uuid/uuid.h> header file. */
 #undef HAVE_UUID_UUID_H
 
+/* Define to 1 if your compiler knows the visibility("hidden") attribute. */
+#undef HAVE_VISIBILITY_ATTRIBUTE
+
 /* Define to 1 if you have the `wcstombs_l' function. */
 #undef HAVE_WCSTOMBS_L
 
diff --git a/src/makefiles/pgxs.mk b/src/makefiles/pgxs.mk
index 0f71fa293d0..7ba8d5bc980 100644
--- a/src/makefiles/pgxs.mk
+++ b/src/makefiles/pgxs.mk
@@ -101,8 +101,11 @@ endif # PGXS
 
 override CPPFLAGS := -I. -I$(srcdir) $(CPPFLAGS)
 
+# See equivalent block in Makefile.shlib
 ifdef MODULES
-override CFLAGS += $(CFLAGS_SL)
+override LDFLAGS_SL += $(CFLAGS_SL_MODULE)
+override CFLAGS += $(CFLAGS_SL) $(CFLAGS_SL_MODULE)
+override CXXFLAGS += $(CFLAGS_SL) $(CXXFLAGS_SL_MODULE)
 endif
 
 ifdef MODULEDIR
diff --git a/src/tools/msvc/Project.pm b/src/tools/msvc/Project.pm
index 570bab563a7..b24a2a98155 100644
--- a/src/tools/msvc/Project.pm
+++ b/src/tools/msvc/Project.pm
@@ -419,13 +419,6 @@ sub Save
 {
 	my ($self) = @_;
 
-	# If doing DLL and haven't specified a DEF file, do a full export of all symbols
-	# in the project.
-	if ($self->{type} eq "dll" && !$self->{def})
-	{
-		$self->FullExportDLL($self->{name} . ".lib");
-	}
-
 	# Warning 4197 is about double exporting, disable this per
 	# http://connect.microsoft.com/VisualStudio/feedback/ViewFeedback.aspx?FeedbackID=99193
 	$self->DisableLinkerWarnings('4197') if ($self->{platform} eq 'x64');
diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm
index fa32dc371dc..f2427008df6 100644
--- a/src/tools/msvc/Solution.pm
+++ b/src/tools/msvc/Solution.pm
@@ -429,6 +429,7 @@ sub GenerateFiles
 		HAVE_WINLDAP_H                           => undef,
 		HAVE_WCSTOMBS_L                          => 1,
 		HAVE_WCTYPE_H                            => 1,
+		HAVE_VISIBILITY_ATTRIBUTE                => undef,
 		HAVE_WRITEV                              => undef,
 		HAVE_X509_GET_SIGNATURE_NID              => 1,
 		HAVE_X86_64_POPCNTQ                      => undef,