Merge pull request #4954 from the-horo/ldc2-conf

Support ldc2.conf as a directory

Author: kinke

.cirrus.yml

@@ -65,12 +65,17 @@ packaging_steps_template: &PACKAGING_STEPS_TEMPLATE
     cd build
     ninja install > /dev/null
     cd ..
-    perl -pi -e s?$PWD/install/?%%ldcbinarypath%%/../?g install/etc/ldc2.conf
+    perl -pi -e s?$PWD/install/?%%ldcbinarypath%%/../?g install/etc/ldc2.conf/*
     if [[ "$CI_OS" == "freebsd" ]]; then
-      perl -pi -e "s?,druntime-ldc\",?,druntime-ldc\", \"-gcc=$CC\",?" install/etc/ldc2.conf
+      cat > install/etc/ldc2.conf/35-ldc-default-CC.conf <<EOF
+    "default":
+    {
+        switches ~= [ "-gcc=${CC}" ];
+    }
+    EOF
     fi
     cp $CIRRUS_WORKING_DIR/{LICENSE,packaging/README} install
-    cat install/etc/ldc2.conf
+    cat install/etc/ldc2.conf/*
     # Now rename the installation dir to test portability
     mv install installed
   # Run hello-world integration test with shared libs

.github/actions/5-install/action.yml

@@ -20,7 +20,7 @@ runs:
         else
           # the cross-compiled runtime libs have already been installed:
           # * lib/: runtime library artifacts
-          # * etc/ldc2.conf
+          # * etc/ldc2.conf/30-ldc-runtime-lib.conf
 
           # now extend by installing the cross-compiled compiler:
           # * bin/: executables
@@ -50,8 +50,9 @@ runs:
           # /d/a/1/install => D:/a/1/install
           absInstallDir=$(cygpath --mixed "$absInstallDir")
         fi
-        perl -pi -e "s|$absInstallDir/|%%ldcbinarypath%%/../|g" install/etc/ldc2.conf
-        cat install/etc/ldc2.conf
+        confs=( install/etc/ldc2.conf/* )
+        perl -pi -e "s|$absInstallDir/|%%ldcbinarypath%%/../|g" "${confs[@]}"
+        cat "${confs[@]}"
 
     - name: Rename the installation dir to test portability
       shell: bash

.github/actions/5a-ios/action.yml

@@ -32,8 +32,7 @@ runs:
           CMAKE_OSX_DEPLOYMENT_TARGET="$deployment_target" \
           BUILD_LTO_LIBS=ON
 
-        section="
-        \"$arch-apple-ios\":
+        section="\"$arch-apple-ios\":
         {
             switches ~= [
                 \"-Xcc=-isysroot\",
@@ -44,5 +43,5 @@ runs:
             ];
             rpath = \"%%ldcbinarypath%%/../lib-ios-$arch\";
         };"
-        echo "$section" >> installed/etc/ldc2.conf
-        cat installed/etc/ldc2.conf
+        echo "$section" >> installed/etc/ldc2.conf/31-ldc-runtime-lib-ios-$arch.conf
+        cat installed/etc/ldc2.conf/*

.github/actions/merge-macos/action.yml

@@ -39,77 +39,26 @@ runs:
         rm lib-{x86_64,arm64}/libLTO.dylib
 
         # ldc2.conf:
-        # 1) make a backup copy
-        cp etc/ldc2.conf /tmp/ldc2.conf.bak
-        # 2) strip to the header comments (remove all existing sections, only keep `default:` line)
-        sed -i '' '/^default:$/q' etc/ldc2.conf
-        # 3) append all sections (except for wasm)
-        cat >>etc/ldc2.conf <<EOF
-        {
-            // default switches injected before all explicit command-line switches
-            switches = [
-                "-defaultlib=phobos2-ldc,druntime-ldc",
-            ];
-            // default switches appended after all explicit command-line switches
-            post-switches = [
-                "-I%%ldcbinarypath%%/../import",
-            ];
-            // default directories to be searched for libraries when linking
-            lib-dirs = [];
-            // default rpath when linking against the shared default libs
-            rpath = "";
-        };
 
-        // macOS:
-
-        "x86_64-apple-":
+        # 1) x86_64 ios config
+        # already present
+        # 2) arm64 ios config
+        cp ../ldc2-arm64/etc/ldc2.conf/31-ldc-runtime-lib-ios-arm64.conf etc/ldc2.conf
+        # 3) x86_64 & arm64 macos
+        rm etc/ldc2.conf/30-ldc-runtime-lib.conf # the old ldc2-x86_64 config
+        for arch in x86_64 arm64; do
+          cat > etc/ldc2.conf/30-ldc-runtime-lib-${arch}.conf <<EOF
+        "${arch}-apple-":
         {
             lib-dirs = [
-                "%%ldcbinarypath%%/../lib-x86_64",
+                "%%ldcbinarypath%%/../lib-${arch}",
             ];
-            rpath = "%%ldcbinarypath%%/../lib-x86_64";
+            rpath = "%%ldcbinarypath%%/../lib-${arch}";
         };
-
-        "arm64-apple-":
-        {
-            lib-dirs = [
-                "%%ldcbinarypath%%/../lib-arm64",
-            ];
-            rpath = "%%ldcbinarypath%%/../lib-arm64";
-        };
-
-        // iOS:
-
-        "x86_64-apple-ios":
-        {
-            switches ~= [
-                "-Xcc=-isysroot",
-                "-Xcc=/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform/Developer/SDKs/iPhoneSimulator.sdk",
-            ];
-            lib-dirs = [
-                "%%ldcbinarypath%%/../lib-ios-x86_64",
-            ];
-            rpath = "%%ldcbinarypath%%/../lib-ios-x86_64";
-        };
-
-        "arm64-apple-ios":
-        {
-            switches ~= [
-                "-Xcc=-isysroot",
-                "-Xcc=/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk",
-            ];
-            lib-dirs = [
-                "%%ldcbinarypath%%/../lib-ios-arm64",
-            ];
-            rpath = "%%ldcbinarypath%%/../lib-ios-arm64";
-        };
-
-        // WebAssembly
         EOF
-        # 4) append the wasm section from the backup
-        sed -n '/^"\^wasm/,$p' /tmp/ldc2.conf.bak | sed '/^\};$/q' >> etc/ldc2.conf
+        done
 
-        cat etc/ldc2.conf
+        cat etc/ldc2.conf/*
 
     - name: Run x86_64/arm64 macOS/iOS cross-compilation smoke tests
       shell: bash

.github/actions/merge-windows/action.yml

@@ -20,23 +20,23 @@ runs:
         mv ldc2-*-x64 ldc2-multilib
         cd ldc2-multilib
         mv lib lib64
+        mv etc/ldc2.conf/30-ldc-runtime-{lib,lib64}.conf
         cp -R ../ldc2-x86/lib lib32
         cp ../ldc2-x86/bin/{*.dll,*.pdb,curl-ca-bundle.crt} lib32/
     - name: Merge ldc2.conf
       shell: pwsh
       run: |
         cd ldc2-multilib
-        (cat etc\ldc2.conf).replace('%%ldcbinarypath%%/../lib', '%%ldcbinarypath%%/../lib64') | Set-Content etc\ldc2.conf
-        $conf32 = '
-        "i[3-6]86-.*-windows-msvc":
+        (cat etc\ldc2.conf\30-ldc-runtime-lib64.conf).replace('%%ldcbinarypath%%/../lib', '%%ldcbinarypath%%/../lib64') | Set-Content etc\ldc2.conf\30-ldc-runtime-lib64.conf
+        $conf32 = '"i[3-6]86-.*-windows-msvc":
         {
             lib-dirs = [
                 "%%ldcbinarypath%%/../lib32",
             ];
         };
         '
-        Add-Content etc\ldc2.conf $conf32 -NoNewline
-        cat etc\ldc2.conf
+        Set-Content etc\ldc2.conf\31-ldc-runtime-lib32.conf $conf32
+        cat etc\ldc2.conf\*
 
     - name: Generate hello.d
       shell: bash
@@ -86,16 +86,15 @@ runs:
       shell: pwsh
       run: |
         cd ldc2-multilib
-        $conf = '
-        "(aarch|arm)64-.*-windows-msvc":
+        $conf = '"(aarch|arm)64-.*-windows-msvc":
         {
             lib-dirs = [
                 "%%ldcbinarypath%%/../libarm64",
             ];
         };
         '
-        Add-Content etc\ldc2.conf $conf -NoNewline
-        cat etc\ldc2.conf
+        Set-Content etc\ldc2.conf\31-ldc-runtime-libarm64.conf $conf
+        cat etc\ldc2.conf\*
     - name: Run arm64 hello-world cross-compilation smoke tests
       shell: cmd
       run: |

CHANGELOG.md

@@ -2,7 +2,9 @@
 
 #### Big news
 - Frontend, druntime and Phobos are at version ~[2.112.0](https://dlang.org/changelog/2.112.0.html), incl. new command-line options `-extI`, `-dllimport=externalOnly` and `-edition`. (#4949, #4962)
-- **Breaking change for dcompute**: The special `@kernel` UDA is now a function and _**requires**_ parentheses as in `@kernel() void foo(){}`. Optionally you can provide launch dimensions, `@kernel([2,4,8])`, to specify to the compute runtime how the kernel is intended to be launched. 
+- **Breaking change for dcompute**: The special `@kernel` UDA is now a function and _**requires**_ parentheses as in `@kernel() void foo(){}`. Optionally you can provide launch dimensions, `@kernel([2,4,8])`, to specify to the compute runtime how the kernel is intended to be launched.
+- ldc2.conf can now be a directory. All the files inside it, ordered naturally, will be concatenated and treated like a big config. (#4954)
+- **Breaking change for ldc2.conf cmake generation**: The `cmake` build process now generates the `ldc2.conf` and `ldc2_install.conf` as directories. `ldc2*.conf.in` and `ADDITIONAL_DEFAULT_LDC_SWITCHES` have been removed, if you need to add switches check out `makeConfSection` in `LdcConfig.cmake`. (#4954)
 
 #### Platform support
 

CMakeLists.txt

@@ -17,23 +17,14 @@ project(ldc)
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules")
 
+include(LdcCommon)
+
 include(FindDCompiler)
 include(CheckCXXCompilerFlag)
 include(CheckDSourceCompiles)
 include(CheckLinkFlag)
 include(BuildDExecutable)
 
-# Helper function
-function(append value)
-    foreach(variable ${ARGN})
-        if(${variable} STREQUAL "")
-            set(${variable} "${value}" PARENT_SCOPE)
-        else()
-            set(${variable} "${${variable}} ${value}" PARENT_SCOPE)
-        endif()
-    endforeach(variable)
-endfunction()
-
 #
 # Locate LLVM.
 #
@@ -128,20 +119,9 @@ set(DMDFE_PATCH_VERSION   0)
 
 set(DMD_VERSION ${DMDFE_MAJOR_VERSION}.${DMDFE_MINOR_VERSION}.${DMDFE_PATCH_VERSION})
 
-# Generally, we want to install everything into CMAKE_INSTALL_PREFIX, but when
-# it is /usr, put the config files into /etc to meet common practice.
-if(NOT DEFINED SYSCONF_INSTALL_DIR)
-    if(CMAKE_INSTALL_PREFIX STREQUAL "/usr")
-        set(SYSCONF_INSTALL_DIR "/etc")
-    else()
-        set(SYSCONF_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/etc")
-    endif()
-endif()
-
 set(D_VERSION ${DMDFE_MAJOR_VERSION} CACHE STRING "D language version")
 set(PROGRAM_PREFIX "" CACHE STRING "Prepended to ldc/ldmd binary names")
 set(PROGRAM_SUFFIX "" CACHE STRING "Appended to ldc/ldmd binary names")
-set(CONF_INST_DIR ${SYSCONF_INSTALL_DIR} CACHE PATH "Directory ldc2.conf is installed to")
 set(INCLUDE_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/include/d CACHE PATH "Path to install D modules to")
 
 # Note: LIB_SUFFIX should perhaps be renamed to LDC_LIBDIR_SUFFIX.
@@ -923,10 +903,6 @@ if(NOT DEFINED COMPILER_RT_LIBDIR_CONFIG)
         set(COMPILER_RT_LIBDIR_CONFIG "${COMPILER_RT_LIBDIR}")
     endif()
 endif()
-if(DEFINED COMPILER_RT_LIBDIR_CONFIG)
-    message(STATUS "Adding ${COMPILER_RT_LIBDIR_CONFIG} to lib-dirs in configuration file")
-    set(OPTIONAL_COMPILER_RT_DIR "\n        \"${COMPILER_RT_LIBDIR_CONFIG}\", // compiler-rt directory")
-endif()
 
 #
 # Auxiliary build and test utils.
@@ -974,6 +950,59 @@ endif()
 #
 add_subdirectory(tools)
 
+#
+# Compiler ldc2.conf configuration
+#
+
+set(switches)
+
+# LLVM 16: Disable function specializations by default.
+# They cause miscompiles of e.g. the frontend for some targets (macOS x86_64 and Windows x64).
+if(LDC_LLVM_VER GREATER 1599 AND LDC_LLVM_VER LESS 1700)
+    list(APPEND switches "-func-specialization-size-threshold=1000000000")
+endif()
+
+if(DEFINED COMPILER_RT_LIBDIR_CONFIG)
+    message(STATUS "Adding ${COMPILER_RT_LIBDIR_CONFIG} to lib-dirs in configuration file")
+endif()
+
+list(APPEND switches "-defaultlib=phobos2-ldc,druntime-ldc")
+
+makeConfSection(NAME "35-ldc-compiler" SECTION "default"
+    BUILD
+    SWITCHES ${switches}
+    # -defaultlib is configure in runtime/CMakeLists.txt
+    POST_SWITCHES
+        "-I${PROJECT_SOURCE_DIR}/runtime/druntime/src"
+        "-I${PROJECT_BINARY_DIR}/import"
+        "-I${PROJECT_SOURCE_DIR}/runtime/phobos"
+        "-I${PROJECT_SOURCE_DIR}/runtime/jit-rt/d"
+    LIB_DIRS ${COMPILER_RT_LIBDIR_CONFIG}
+
+    INSTALL
+    SWITCHES ${switches}
+    POST_SWITCHES "-I${INCLUDE_INSTALL_DIR}"
+    LIB_DIRS ${COMPILER_RT_LIBDIR_CONFIG}
+)
+
+set(wasm_switches)
+list(APPEND wasm_switches -defaultlib=)
+# Default wasm stack is only 64kb, this is rather small, let's bump it to 1mb
+list(APPEND wasm_switches -L-z -Lstack-size=1048576)
+# Protect from stack overflow overwriting global memory
+list(APPEND wasm_switches -L--stack-first)
+if(LDC_WITH_LLD)
+    list(APPEND wasm_switches -link-internally)
+endif()
+# LLD 8+ requires (new) `--export-dynamic` for WebAssembly (https://github.com/ldc-developers/ldc/issues/3023).
+list(APPEND wasm_switches -L--export-dynamic)
+
+makeConfSection(NAME "40-ldc-wasm"
+    SECTION "^wasm(32|64)-"
+    SWITCHES ${wasm_switches}
+    LIB_DIRS OVERRIDE
+)
+
 #
 # Test and runtime targets. Note that enable_testing() is order-sensitive!
 #
@@ -1006,6 +1035,15 @@ option(LDC_BUILD_RUNTIME "Build the runtime libraries" ${_LDC_BUILD_RUNTIME_DEFA
 
 if(LDC_BUILD_RUNTIME)
     add_subdirectory(runtime)
+
+    # POST_BUILD can't be used for targets in a different directory so
+    # runtime/CMakeLists.txt can't directly add the commands.
+    if(_LDC_POST_BUILD_COMMANDS)
+        add_custom_command(TARGET ${LDC_EXE} POST_BUILD
+            ${_LDC_POST_BUILD_COMMANDS}
+            BYPRODUCTS ${_LDC_POST_BUILD_BYPRODUCTS}
+        )
+    endif()
 else()
     message(STATUS "NOT building the runtime libraries (LDC_BUILD_RUNTIME=OFF)")
 endif()
@@ -1045,6 +1083,11 @@ if(${BUILD_SHARED})
     install(TARGETS ${LDC_LIB} DESTINATION ${CMAKE_INSTALL_PREFIX}/lib${LIB_SUFFIX})
 endif()
 
+# Add some documentation to the installed config files
+if(NOT DONT_INSTALL_CONF)
+    install(FILES "ldc2.conf.header" DESTINATION "${CONF_INST_DIR}/ldc2.conf" RENAME 00-docs.conf)
+endif()
+
 if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
     if(NOT DEFINED BASH_COMPLETION_COMPLETIONSDIR)
         find_package(bash-completion QUIET)

cmake/Modules/LdcCommon.cmake

@@ -0,0 +1,13 @@
+# Common functionality shared by the compiler and the runtime build system.
+
+function(append value)
+    foreach(variable ${ARGN})
+        if(${variable} STREQUAL "")
+            set(${variable} "${value}" PARENT_SCOPE)
+        else()
+            set(${variable} "${${variable}} ${value}" PARENT_SCOPE)
+        endif()
+    endforeach(variable)
+endfunction()
+
+include(LdcConfig)