Merge pull request #368 from fortran-lang/feat/generic-macro-expansions

feat(pp): adds preprocessor macro expansion

Author: gnikit

CHANGELOG.md

@@ -6,6 +6,10 @@
 
 ### Added
 
+- Added support for preprocessor macro expansions
+  ([#368](https://github.com/fortran-lang/fortls/pull/368))
+- Added support for leading white spaces in preprocessor directives
+  ([#297](https://github.com/fortran-lang/fortls/issues/297))
 - Added hover messages for Types and Modules
   ([#208](https://github.com/fortran-lang/fortls/issues/208))
 - Added support for Markdown intrinsics from the M_intrinsics repository

fortls/langserver.py

@@ -751,11 +751,18 @@ def get_definition(
             return None
         # Search in Preprocessor defined variables
         if def_name in def_file.pp_defs:
+            def_value = def_file.pp_defs.get(def_name)
+            def_arg_str = ""
+            if isinstance(def_value, tuple):
+                def_arg_str, def_value = def_value
+                def_arg_str = ", ".join([x.strip() for x in def_arg_str.split(",")])
+                def_arg_str = f"({def_arg_str})"
+
             var = Variable(
                 def_file.ast,
                 def_line + 1,
                 def_name,
-                f"#define {def_name} {def_file.pp_defs.get(def_name)}",
+                f"#define {def_name}{def_arg_str} {def_value}",
                 [],
             )
             return var

fortls/parsers/internal/parser.py

@@ -2079,6 +2079,23 @@ def replace_vars(line: str):
         else:
             return line_res
 
+    def expand_func_macro(def_name: str, def_value: tuple[str, str]):
+        def_args, sub = def_value
+        def_args = def_args.split(",")
+        regex = re.compile(rf"\b{def_name}\s*\({','.join(['(.*)']*len(def_args))}\)")
+
+        for i, arg in enumerate(def_args, start=1):
+            sub = re.sub(rf"\b({arg.strip()})\b", rf"\\{i}", sub)
+
+        return regex, sub
+
+    def append_multiline_macro(def_value: str | tuple, line: str):
+        if isinstance(def_value, tuple):
+            def_args, def_value = def_value
+            def_value += line
+            return (def_args, def_value)
+        return def_value + line
+
     if pp_defs is None:
         pp_defs = {}
     if include_dirs is None:
@@ -2097,11 +2114,13 @@ def replace_vars(line: str):
         # Handle multiline macro continuation
         if def_cont_name is not None:
             output_file.append("")
-            if line.rstrip()[-1] != "\\":
-                defs_tmp[def_cont_name] += line.strip()
+            is_multiline = line.strip()[-1] != "\\"
+            line_to_append = line.strip() if is_multiline else line[0:-1].strip()
+            defs_tmp[def_cont_name] = append_multiline_macro(
+                defs_tmp[def_cont_name], line_to_append
+            )
+            if is_multiline:
                 def_cont_name = None
-            else:
-                defs_tmp[def_cont_name] += line[0:-1].strip()
             continue
         # Handle conditional statements
         match = FRegex.PP_REGEX.match(line)
@@ -2110,14 +2129,14 @@ def replace_vars(line: str):
             def_name = None
             if_start = False
             # Opening conditional statements
-            if match.group(1) == "if ":
+            if match.group(1).lower() == "if ":
                 is_path = eval_pp_if(line[match.end(1) :], defs_tmp)
                 if_start = True
-            elif match.group(1) == "ifdef":
+            elif match.group(1).lower() == "ifdef":
                 if_start = True
                 def_name = line[match.end(0) :].strip()
                 is_path = def_name in defs_tmp
-            elif match.group(1) == "ifndef":
+            elif match.group(1).lower() == "ifndef":
                 if_start = True
                 def_name = line[match.end(0) :].strip()
                 is_path = not (def_name in defs_tmp)
@@ -2135,7 +2154,7 @@ def replace_vars(line: str):
             inc_start = False
             exc_start = False
             exc_continue = False
-            if match.group(1) == "elif":
+            if match.group(1).lower() == "elif":
                 if (not pp_stack_group) or (pp_stack_group[-1][0] != len(pp_stack)):
                     # First elif statement for this elif group
                     if pp_stack[-1][0] < 0:
@@ -2155,7 +2174,7 @@ def replace_vars(line: str):
                     inc_start = True
                 else:
                     exc_start = True
-            elif match.group(1) == "else":
+            elif match.group(1).lower() == "else":
                 if pp_stack[-1][0] < 0:
                     pp_stack[-1][0] = i + 1
                     exc_start = True
@@ -2171,7 +2190,7 @@ def replace_vars(line: str):
                     pp_skips.append(pp_stack.pop())
                     pp_stack.append([-1, -1])
                     inc_start = True
-            elif match.group(1) == "endif":
+            elif match.group(1).lower() == "endif":
                 if pp_stack_group and (pp_stack_group[-1][0] == len(pp_stack)):
                     pp_stack_group.pop()
                 if pp_stack[-1][0] < 0:
@@ -2209,12 +2228,18 @@ def replace_vars(line: str):
                 if eq_ind >= 0:
                     # Handle multiline macros
                     if line.rstrip()[-1] == "\\":
-                        defs_tmp[def_name] = line[match.end(0) + eq_ind : -1].strip()
+                        def_value = line[match.end(0) + eq_ind : -1].strip()
                         def_cont_name = def_name
                     else:
-                        defs_tmp[def_name] = line[match.end(0) + eq_ind :].strip()
+                        def_value = line[match.end(0) + eq_ind :].strip()
                 else:
-                    defs_tmp[def_name] = "True"
+                    def_value = "True"
+
+                # are there arguments to parse?
+                if match.group(3):
+                    def_value = (match.group(4), def_value)
+
+                defs_tmp[def_name] = def_value
             elif (match.group(1) == "undef") and (def_name in defs_tmp):
                 defs_tmp.pop(def_name, None)
             log.debug(f"{line.strip()} !!! Define statement({i + 1})")
@@ -2265,8 +2290,15 @@ def replace_vars(line: str):
                 continue
             def_regex = def_regexes.get(def_tmp)
             if def_regex is None:
-                def_regex = re.compile(rf"\b{def_tmp}\b")
+                if isinstance(value, tuple):
+                    def_regex = expand_func_macro(def_tmp, value)
+                else:
+                    def_regex = re.compile(rf"\b{def_tmp}\b")
                 def_regexes[def_tmp] = def_regex
+
+            if isinstance(def_regex, tuple):
+                def_regex, value = def_regex
+
             line_new, nsubs = def_regex.subn(value, line)
             if nsubs > 0:
                 log.debug(

fortls/regex_patterns.py

@@ -124,11 +124,14 @@ class FortranRegularExpressions:
     FREE_FORMAT_TEST: Pattern = compile(r"[ ]{1,4}[a-z]", I)
     # Preprocessor matching rules
     DEFINED: Pattern = compile(r"defined[ ]*\(?[ ]*([a-z_]\w*)[ ]*\)?", I)
-    PP_REGEX: Pattern = compile(r"#(if |ifdef|ifndef|else|elif|endif)")
-    PP_DEF: Pattern = compile(r"#(define|undef)[ ]*([\w]+)(\((\w+(,[ ]*)?)+\))?", I)
+    PP_REGEX: Pattern = compile(r"[ ]*#[ ]*(if |ifdef|ifndef|else|elif|endif)", I)
+    PP_DEF: Pattern = compile(
+        r"[ ]*#[ ]*(define|undef|undefined)[ ]*(\w+)(\([ ]*([ \w,]*?)[ ]*\))?",
+        I,
+    )
     PP_DEF_TEST: Pattern = compile(r"(![ ]*)?defined[ ]*\([ ]*(\w*)[ ]*\)$", I)
-    PP_INCLUDE: Pattern = compile(r"#include[ ]*([\"\w\.]*)", I)
-    PP_ANY: Pattern = compile(r"(^#:?\w+)")
+    PP_INCLUDE: Pattern = compile(r"[ ]*#[ ]*include[ ]*([\"\w\.]*)", I)
+    PP_ANY: Pattern = compile(r"^[ ]*#:?[ ]*(\w+)")
     # Context matching rules
     CALL: Pattern = compile(r"[ ]*CALL[ ]+[\w%]*$", I)
     INT_STMNT: Pattern = compile(r"^[ ]*[a-z]*$", I)

test/test_preproc.py

@@ -52,12 +52,12 @@ def check_return(result_array, checks):
         "```fortran90\n#define PETSC_ERR_INT_OVERFLOW 84\n```",
         "```fortran90\n#define varVar 55\n```",
         (
-            "```fortran90\n#define ewrite if (priority <= 3) write((priority),"
-            " format)\n```"
+            "```fortran90\n#define ewrite(priority, format)"
+            " if (priority <= 3) write((priority), format)\n```"
         ),
         (
-            "```fortran90\n#define ewrite2 if (priority <= 3) write((priority),"
-            " format)\n```"
+            "```fortran90\n#define ewrite2(priority, format)"
+            " if (priority <= 3) write((priority), format)\n```"
         ),
         "```fortran90\n#define SUCCESS .true.\n```",
         "```fortran90\nREAL, CONTIGUOUS, POINTER, DIMENSION(:) :: var1\n```",

test/test_preproc_parser.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from fortls.parsers.internal.parser import preprocess_file
+
+
+def test_pp_leading_spaces():
+    lines = [
+        " #define LEADING_SPACES_INDENT 1",
+        "  #   define LEADING_SPACES_INDENT2",
+        "    #    define FILE_ENCODING ,encoding='UTF-8'",
+        "program pp_intentation",
+        "  implicit none",
+        "  print*, LEADING_SPACES_INDENT",
+        "  open(unit=1,file='somefile.txt' FILE_ENCODING)",
+        "end program pp_intentation",
+    ]
+    _, _, _, defs = preprocess_file(lines)
+    ref = {
+        "LEADING_SPACES_INDENT": "1",
+        "LEADING_SPACES_INDENT2": "True",
+        "FILE_ENCODING": ",encoding='UTF-8'",
+    }
+    assert defs == ref
+
+
+def test_pp_macro_expansion():
+    lines = [
+        "# define WRAP(PROCEDURE) PROCEDURE , wrap_/**/PROCEDURE",
+        "generic, public :: set => WRAP(abc)",
+        "procedure :: WRAP(abc)",
+    ]
+    ref = [
+        "# define WRAP(PROCEDURE) PROCEDURE , wrap_/**/PROCEDURE",
+        "generic, public :: set => abc , wrap_/**/abc",
+        "procedure :: abc , wrap_/**/abc",
+    ]
+    output, _, _, _ = preprocess_file(lines)
+    assert output == ref