Fix broken static linking of apps.

Use pkg-config for both internal and external builds of
libevent, hwloc, and libpmix and store them in
OPAL_WRAPPER_EXTRA_LIBS.

Signed-off-by: Austen Lauria <awlauria@us.ibm.com>

Author: awlauria

config/opal_check_pc.m4

@@ -0,0 +1,105 @@
+dnl -*- shell-script -*-
+dnl
+dnl Copyright (c) 2021 IBM Corporation.  All rights reserved.
+dnl
+dnl $COPYRIGHT$
+dnl
+dnl Additional copyrights may follow
+dnl
+dnl $HEADER$
+dnl
+
+
+dnl OPAL_CHECK_PKG_CONFIG
+dnl Check for availability of pkg-config and store the result.
+dnl If it is not available, store any passed in libs from the
+dnl --with-extra-libs configure option, or the known defaults.
+dnl
+dnl If it is available, allow configury to check for .pc files
+dnl and append to OPAL_WRAPPER_EXTRA_LIBS.
+AC_DEFUN([OPAL_CHECK_PKG_CONFIG], [
+  AS_IF([test -z "$OPAL_HAS_PKG_CONFIG"],
+        [
+          AC_MSG_CHECKING([If pkg-config is available])
+          which pkg-config > /dev/null 2>&1
+          AS_IF([test "$?" -eq "0"],
+                [
+                  AC_MSG_RESULT([yes])
+                  OPAL_HAS_PKG_CONFIG="yes"
+                ],
+                [
+                  AC_MSG_RESULT([no])
+                  OPAL_HAS_PKG_CONFIG="no"
+                  OPAL_FLAGS_APPEND_UNIQ([OPAL_WRAPPER_EXTRA_LIBS], [$OPAL_EXTRA_LIBS])
+                ]
+          )
+        ]
+  )
+])
+
+dnl OPAL_GET_LFLAGS_FROM_PC
+dnl Get the -l flags using pkg-config on the passed in .pc file, and
+dnl append them to OPAL_WRAPPER_EXTRA_LIBS.
+dnl
+dnl Requires 1 or more arguments.
+dnl
+dnl Arg 1 (Required): the name of the .pc file to look for (hwloc/pmix/libevent_core).pc
+dnl 
+dnl Arg 2..n (Optional): The search path for the .pc file above and any dependencies.
+dnl                      These search will be appended and set in the env variable
+dnl                      PKG_CONFIG_PATH to tell pkg-config where to locate the .pc
+dnl                      file from argument 1, and its dependencies.
+dnl
+AC_DEFUN([OPAL_GET_LFLAGS_FROM_PC], [
+
+  AC_REQUIRE([OPAL_CHECK_PKG_CONFIG])
+  AS_IF([test "$OPAL_HAS_PKG_CONFIG" = "yes"],
+        [
+          AC_MSG_NOTICE([Looking for pc file for $1])
+          OPAL_VAR_SCOPE_PUSH([opal_pkg_config_name opal_pkg_config_env_to_set opal_pkg_config_args opal_pkg_config_cmd opal_pkg_config_result])
+          opal_pkg_config_name=$1
+          opal_pkg_config_env_to_set=""
+          opal_pkg_config_cmd=""
+
+          AS_LITERAL_WORD_IF([$1], [], [m4_fatal([Non-literal argument $1])], [])
+          $1_OMPI_PC_DIR=""
+
+          # Tell pkg-config where to find the .pc file from argument 1
+          # as well as any/all dependencies.
+          # The following block will append the path
+          # to those .pc files into the pkg-config command.
+          # num_paths > 1 implies there is at least one dependent .pc file
+          # to append.
+          AS_IF([test $# -gt 1],
+                [
+                  $1_OMPI_PC_DIR=$2
+
+                  # Shift the arguments by one to get to the actual paths.
+                  opal_pkg_config_args=m4_shift($@)
+
+                  # Iterate over the comma seperated arguments, and replace the ','
+                  for i in $(echo $opal_pkg_config_args | sed "s/,/ /g"); do
+                    opal_pkg_config_env_to_set="$i:${opal_pkg_config_env_to_set}"
+                  done
+                  opal_pkg_config_cmd="env PKG_CONFIG_PATH=$opal_pkg_config_env_to_set pkg-config --libs $opal_pkg_config_name"
+                ],
+                [opal_pkg_config_cmd="pkg-config --libs $opal_pkg_config_name"]
+          )
+
+          OPAL_LOG_MSG([pkg-config cmd for $opal_pkg_config_name: $opal_pkg_config_cmd])
+
+          opal_pkg_config_result=$($opal_pkg_config_cmd)
+          AS_IF([test -z "$opal_pkg_config_result"],
+                [
+                  AC_MSG_WARN([Could not find viable $opal_pkg_config_name.pc])
+                ],
+                [
+                  AC_MSG_NOTICE([pkg-config result $opal_pkg_config_result])
+                  OPAL_FLAGS_APPEND_UNIQ([OPAL_WRAPPER_EXTRA_LIBS], [$opal_pkg_config_result])
+                ]
+          )
+          OPAL_VAR_SCOPE_POP
+        ],
+        []
+  )
+])

config/opal_config_hwloc.m4

@@ -70,6 +70,14 @@ AC_DEFUN([OPAL_CONFIG_HWLOC], [
     AS_IF([test "$external_hwloc_happy" = "0" -a "$internal_hwloc_happy" = "0"],
           [AC_MSG_ERROR([Could not find viable hwloc build.])])
 
+    AS_IF([test "$opal_hwloc_mode" = "external"],
+        [AS_IF([test -n "$with_hwloc"],
+               [OPAL_GET_LFLAGS_FROM_PC(hwloc, $with_hwloc/lib/pkgconfig)],
+               [OPAL_GET_LFLAGS_FROM_PC(hwloc)]
+         )],
+        [OPAL_GET_LFLAGS_FROM_PC(hwloc, $OMPI_TOP_SRCDIR/3rd-party/hwloc_directory)]
+    )
+
     # this will work even if there is no hwloc package included,
     # because hwloc_tarball and hwloc_directory will evaluate to an
     # empty string.  These are relative to the 3rd-party/ directory.

config/opal_config_libevent.m4

@@ -70,6 +70,31 @@ AC_DEFUN([OPAL_CONFIG_LIBEVENT], [
     AS_IF([test "$external_libevent_happy" = "0" -a "$internal_libevent_happy" = "0"],
           [AC_MSG_ERROR([Could not find viable libevent build.])])
 
+    AS_IF([test "$opal_libevent_mode" = "external"],
+        # The libevent.pc file only returns -levent, however we want
+        # only -libevent_core and -libevent_pthreads, so ompi needs to
+        # do this twice, once for each.
+        [AS_IF([test -n "$with_libevent"],
+               [
+                 OPAL_GET_LFLAGS_FROM_PC(libevent_core, $with_libevent/lib/pkgconfig)
+                 OPAL_GET_LFLAGS_FROM_PC(libevent_pthreads, $with_libevent/lib/pkgconfig)
+               ],
+               [
+                 OPAL_GET_LFLAGS_FROM_PC(libevent_core)
+                 OPAL_GET_LFLAGS_FROM_PC(libevent_pthreads)
+               ]
+         )],
+        [
+          OPAL_GET_LFLAGS_FROM_PC(libevent_core, $OMPI_TOP_SRCDIR/3rd-party/libevent_directory)
+          OPAL_GET_LFLAGS_FROM_PC(libevent_pthreads, $OMPI_TOP_SRCDIR/3rd-party/libevent_directory)
+        ]
+    )
+
+    # Strip -levent as we don't want it. We only want -levent_core/-levent_pthreads.
+    # See https://github.com/open-mpi/ompi/pull/8792
+    OPAL_WRAPPER_EXTRA_LIBS=$(echo $OPAL_WRAPPER_EXTRA_LIBS | sed "s/\\-levent\b//g");
+    AC_MSG_NOTICE([OPAL_WRAPPER_EXTRA_LIBS stripped -levent: $OPAL_WRAPPER_EXTRA_LIBS])
+
     # this will work even if there is no libevent package included,
     # because libevent_tarball and libevent_directory will evaluate to
     # an empty string.  These are relative to the 3rd-party/

config/opal_config_pmix.m4

@@ -129,6 +129,14 @@ AC_DEFUN([OPAL_CONFIG_PMIX], [
     AS_IF([test "$external_pmix_happy" = "0" -a "$internal_pmix_happy" = "0"],
           [AC_MSG_ERROR([Could not find viable pmix build.])])
 
+   AS_IF([test "$external_pmix_happy" = "1"],
+         [AS_IF([test -n "$with_pmix"],
+               [OPAL_GET_LFLAGS_FROM_PC(pmix, $with_pmix/lib/pkgconfig, $libevent_core_OMPI_PC_DIR, $hwloc_OMPI_PC_DIR)],
+               [OPAL_GET_LFLAGS_FROM_PC(pmix, "", $libevent_core_OMPI_PC_DIR, $hwloc_OMPI_PC_DIR)]
+         )],
+        [OPAL_GET_LFLAGS_FROM_PC(pmix, $OMPI_TOP_SRCDIR/3rd-party/openpmix/maint, $libevent_core_OMPI_PC_DIR, $hwloc_OMPI_PC_DIR)]
+    )
+
     AC_DEFINE_UNQUOTED([OPAL_USING_INTERNAL_PMIX],
                        [$OPAL_USING_INTERNAL_PMIX],
                        [Whether or not we are using the internal PMIx])

config/opal_configure_options.m4

@@ -213,6 +213,22 @@ else
 fi
 AM_CONDITIONAL(WANT_INSTALL_HEADERS, test "$WANT_INSTALL_HEADERS" = 1)
 
+#
+# If there's no pkg-config available, allow users to pass in -l/-L flags manually.
+#
+AC_MSG_CHECKING([if want extra libs for static builds])
+AC_ARG_WITH([extra-libs],[AS_HELP_STRING([--with-extra-libs],
+                  [A comma seperated list of extra libraries to link in for static builds (example: -lbar,-lbat) (default: None.). This should be used if pkg-config is not available.])])
+    OPAL_EXTRA_LIBS=""
+if test -n "x$with_extra_libs"; then
+    # Iterate over the list, removing the commas.
+    for i in $(echo $with_extra_libs | sed "s/,/ /g"); do
+      OPAL_FLAGS_APPEND_UNIQ([OPAL_EXTRA_LIBS], [$i])
+    done
+    AC_MSG_RESULT($OPAL_EXTRA_LIBS)
+else
+    AC_MSG_RESULT(default)
+fi
 
 #
 # Do we want the pretty-print stack trace feature?

config/opal_setup_wrappers.m4

@@ -380,7 +380,7 @@ AC_DEFUN([OPAL_SETUP_WRAPPER_FINAL],[
        # and this cleans the duplication up a bunch.  Always add everything the user
        # asked for, as they know better than us.
        AC_MSG_CHECKING([for OPAL LIBS])
-       OPAL_WRAPPER_EXTRA_LIBS="$opal_mca_wrapper_extra_libs"
+       OPAL_WRAPPER_EXTRA_LIBS="$opal_mca_wrapper_extra_libs $OPAL_WRAPPER_EXTRA_LIBS"
        OPAL_FLAGS_APPEND_UNIQ([OPAL_WRAPPER_EXTRA_LIBS], [$wrapper_extra_libs])
        OPAL_WRAPPER_EXTRA_LIBS="$OPAL_WRAPPER_EXTRA_LIBS $with_wrapper_libs"
        AC_SUBST([OPAL_WRAPPER_EXTRA_LIBS])

ompi/tools/wrappers/mpic++-wrapper-data.txt.in

@@ -21,7 +21,7 @@ linker_flags=@OMPI_WRAPPER_EXTRA_LDFLAGS@
 # pulled in implicitly) because we intend MPI applications to only use
 # the MPI API.
 libs=@OMPI_WRAPPER_CXX_LIB@ -l@OMPI_LIBMPI_NAME@
-libs_static=@OMPI_WRAPPER_CXX_LIB@ -l@OMPI_LIBMPI_NAME@ -l@OPAL_LIB_NAME@ @OMPI_WRAPPER_EXTRA_LIBS@
+libs_static=@OMPI_WRAPPER_CXX_LIB@ -l@OMPI_LIBMPI_NAME@ -l@OPAL_LIB_NAME@ @OMPI_WRAPPER_EXTRA_LIBS@ @OPAL_WRAPPER_EXTRA_LIBS@
 dyn_lib_file=lib@OMPI_LIBMPI_NAME@.@OPAL_DYN_LIB_SUFFIX@
 static_lib_file=lib@OMPI_LIBMPI_NAME@.a
 required_file=@OMPI_WRAPPER_CXX_REQUIRED_FILE@

ompi/tools/wrappers/mpicc-wrapper-data.txt.in

@@ -21,7 +21,7 @@ linker_flags=@OMPI_WRAPPER_EXTRA_LDFLAGS@
 # pulled in implicitly) because we intend MPI applications to only use
 # the MPI API.
 libs=-l@OMPI_LIBMPI_NAME@
-libs_static=-l@OMPI_LIBMPI_NAME@ -l@OPAL_LIB_NAME@ @OMPI_WRAPPER_EXTRA_LIBS@
+libs_static=-l@OMPI_LIBMPI_NAME@ -l@OPAL_LIB_NAME@ @OMPI_WRAPPER_EXTRA_LIBS@ @OPAL_WRAPPER_EXTRA_LIBS@
 dyn_lib_file=lib@OMPI_LIBMPI_NAME@.@OPAL_DYN_LIB_SUFFIX@
 static_lib_file=lib@OMPI_LIBMPI_NAME@.a
 required_file=

ompi/tools/wrappers/mpifort-wrapper-data.txt.in

@@ -20,7 +20,7 @@ linker_flags=@OMPI_WRAPPER_EXTRA_FC_LDFLAGS@
 # pulled in implicitly) because we intend MPI applications to only use
 # the MPI API.
 libs=@OMPI_FORTRAN_USEMPIF08_LIB@ @OMPI_FORTRAN_USEMPI_LIB@ @OMPI_FORTRAN_MPIFH_LINK@ -l@OMPI_LIBMPI_NAME@
-libs_static=@OMPI_FORTRAN_USEMPIF08_LIB@ @OMPI_FORTRAN_USEMPI_LIB@ @OMPI_FORTRAN_MPIFH_LINK@ -l@OMPI_LIBMPI_NAME@ -l@OPAL_LIB_NAME@ @OMPI_WRAPPER_EXTRA_LIBS@
+libs_static=@OMPI_FORTRAN_USEMPIF08_LIB@ @OMPI_FORTRAN_USEMPI_LIB@ @OMPI_FORTRAN_MPIFH_LINK@ -l@OMPI_LIBMPI_NAME@ -l@OPAL_LIB_NAME@ @OMPI_WRAPPER_EXTRA_LIBS@ @OPAL_WRAPPER_EXTRA_LIBS@
 dyn_lib_file=lib@OMPI_LIBMPI_NAME@.@OPAL_DYN_LIB_SUFFIX@
 static_lib_file=lib@OMPI_LIBMPI_NAME@.a
 required_file=@OMPI_WRAPPER_FORTRAN_REQUIRED_FILE@

ompi/tools/wrappers/ompi-c.pc.in

@@ -16,6 +16,6 @@ pkgincludedir=@opalincludedir@
 # static linking, so only list these in Libs.private.
 #
 Libs: -L${libdir} @OMPI_PKG_CONFIG_LDFLAGS@ -l@OMPI_LIBMPI_NAME@
-Libs.private: -l@OPAL_LIB_NAME@ @OMPI_WRAPPER_EXTRA_LIBS@
+Libs.private: -l@OPAL_LIB_NAME@ @OMPI_WRAPPER_EXTRA_LIBS@ @OPAL_WRAPPER_EXTRA_LIBS@
 #
 Cflags: -I${includedir} @OMPI_WRAPPER_EXTRA_CPPFLAGS@ @OMPI_WRAPPER_EXTRA_CFLAGS@