Merge pull request #36283 from ahoppen/pr/swiftsyntax-null-char-in-file

[libSyntax] Explicitly pass source file length to c parse actions

Author: ahoppen

include/swift-c/SyntaxParser/SwiftSyntaxParser.h

@@ -218,8 +218,10 @@ swiftparse_parser_set_node_lookup(swiftparse_parser_t,
 /// via the return value of \c swiftparse_parse_string.
 ///
 /// \param source a null-terminated UTF8 string buffer.
+/// \param len The length of the source string. This allows \p source to contain
+///            intermediate null characters.
 SWIFTPARSE_PUBLIC swiftparse_client_node_t
-swiftparse_parse_string(swiftparse_parser_t, const char *source);
+swiftparse_parse_string(swiftparse_parser_t, const char *source, size_t len);
 
 /// Returns a constant string pointer for verification purposes.
 ///

tools/libSwiftSyntaxParser/libSwiftSyntaxParser.cpp

@@ -95,7 +95,7 @@ class SynParser {
     setDiagnosticHandler(nullptr);
   }
 
-  swiftparse_client_node_t parse(const char *source);
+  swiftparse_client_node_t parse(const char *source, size_t len);
 };
 
 class CLibParseActions : public SyntaxParseActions {
@@ -274,10 +274,10 @@ struct SynParserDiagConsumer: public DiagnosticConsumer {
   }
 };
 
-swiftparse_client_node_t SynParser::parse(const char *source) {
+swiftparse_client_node_t SynParser::parse(const char *source, size_t len) {
   SourceManager SM;
-  unsigned bufID = SM.addNewSourceBuffer(
-    llvm::MemoryBuffer::getMemBuffer(source, "syntax_parse_source"));
+  unsigned bufID = SM.addNewSourceBuffer(llvm::MemoryBuffer::getMemBuffer(
+      StringRef(source, len), "syntax_parse_source"));
   TypeCheckerOptions tyckOpts;
   LangOptions langOpts;
   langOpts.BuildSyntaxTree = true;
@@ -329,10 +329,11 @@ swiftparse_parser_set_node_lookup(swiftparse_parser_t c_parser,
   parser->setNodeLookup(lookup);
 }
 
-swiftparse_client_node_t
-swiftparse_parse_string(swiftparse_parser_t c_parser, const char *source) {
+swiftparse_client_node_t swiftparse_parse_string(swiftparse_parser_t c_parser,
+                                                 const char *source,
+                                                 size_t len) {
   SynParser *parser = static_cast<SynParser*>(c_parser);
-  return parser->parse(source);
+  return parser->parse(source, len);
 }
 
 const char* swiftparse_syntax_structure_versioning_identifier(void) {

tools/swift-syntax-parser-test/swift-syntax-parser-test.cpp

@@ -136,17 +136,18 @@ makeNode(const swiftparse_syntax_node_t *raw_node, StringRef source) {
 }
 
 static swiftparse_client_node_t
-parse(const char *source, swiftparse_node_handler_t node_handler,
+parse(StringRef source, swiftparse_node_handler_t node_handler,
       swiftparse_diagnostic_handler_t diag_handler = nullptr) {
   swiftparse_parser_t parser = swiftparse_parser_create();
   swiftparse_parser_set_node_handler(parser, node_handler);
   swiftparse_parser_set_diagnostic_handler(parser, diag_handler);
-  swiftparse_client_node_t top = swiftparse_parse_string(parser, source);
+  swiftparse_client_node_t top =
+      swiftparse_parse_string(parser, source.data(), source.size());
   swiftparse_parser_dispose(parser);
   return top;
 }
 
-static int dumpTree(const char *source) {
+static int dumpTree(StringRef source) {
   swiftparse_node_handler_t nodeHandler =
     ^swiftparse_client_node_t(const swiftparse_syntax_node_t *raw_node) {
       return makeNode(raw_node, source);
@@ -216,7 +217,7 @@ static void printDiagInfo(const swiftparser_diagnostic_t diag,
   }
 }
 
-static int dumpDiagnostics(const char* source, llvm::SourceMgr &SM,
+static int dumpDiagnostics(StringRef source, llvm::SourceMgr &SM,
                            unsigned BufferId) {
   swiftparse_node_handler_t nodeHandler =
   ^swiftparse_client_node_t(const swiftparse_syntax_node_t *raw_node) {
@@ -255,7 +256,7 @@ static void printTimeRecord(unsigned numInvoks, const TimeRecord &total,
   OS << '\n';
 }
 
-static int timeParsing(const char *source, unsigned numInvoks) {
+static int timeParsing(StringRef source, unsigned numInvoks) {
   swiftparse_node_handler_t nodeHandler =
     ^swiftparse_client_node_t(const swiftparse_syntax_node_t *raw_node) {
       return nullptr;
@@ -288,10 +289,10 @@ int main(int argc, char *argv[]) {
   auto BufferId = SM.AddNewSourceBuffer(std::move(*fileBufOrErr), SMLoc());
   switch (options::Action) {
   case ActionType::DumpTree:
-    return dumpTree(source.data());
+    return dumpTree(source);
   case ActionType::Time:
-    return timeParsing(source.data(), options::NumParses);
+    return timeParsing(source, options::NumParses);
   case ActionType::Diagnostics:
-    return dumpDiagnostics(source.data(), SM, BufferId);
+    return dumpDiagnostics(source, SM, BufferId);
   }
 }

unittests/SyntaxParser/SyntaxParserTests.cpp

@@ -19,21 +19,21 @@
 using namespace swift;
 
 static swiftparse_client_node_t
-parse(const char *source, swiftparse_node_handler_t node_handler,
+parse(StringRef source, swiftparse_node_handler_t node_handler,
       swiftparse_node_lookup_t node_lookup) {
   swiftparse_parser_t parser = swiftparse_parser_create();
   swiftparse_parser_set_node_handler(parser, node_handler);
   swiftparse_parser_set_node_lookup(parser, node_lookup);
-  swiftparse_client_node_t top = swiftparse_parse_string(parser, source);
+  swiftparse_client_node_t top = swiftparse_parse_string(parser, source.data(), source.size());
   swiftparse_parser_dispose(parser);
   return top;
 }
 
 TEST(SwiftSyntaxParserTests, IncrementalParsing) {
-  const char *source1 =
+  StringRef source1 =
   "func t1() { }\n"
   "func t2() { }\n";
-  const char *source2 =
+  StringRef source2 =
   "func t1renamed() { }\n"
   "func t2() { }\n";