CSV import options for DELIMITER, NULL, and QUOTE (#256)

Greenscreen23 · web-flow · commit 3456f974582f · 2025-09-17T10:57:56.000+02:00
This branch implements more csv parsing options. The goal is that the
following sql string is correctly parsed:
```
COPY "{}" FROM '{}' WITH (FORMAT CSV, DELIMITER '|', NULL '', QUOTE '"');
```

where delimiter is the delimiter of the csv file, null is the value of
null values in the csv and quote is the string used to quote values.
diff --git a/src/parser/bison_parser.cpp b/src/parser/bison_parser.cpp
diff --git a/src/parser/bison_parser.h b/src/parser/bison_parser.h
@@ -345,10 +345,11 @@ union HSQL_STYPE
   hsql::RowLockWaitPolicy lock_wait_policy_t;
 
   hsql::ImportExportOptions* import_export_option_t;
+  std::pair<hsql::CsvOptionType, char*>* csv_option_t;
 
   // clang-format off
 
-#line 352 "bison_parser.h"
+#line 353 "bison_parser.h"
 
 };
 typedef union HSQL_STYPE HSQL_STYPE;
diff --git a/src/parser/bison_parser.y b/src/parser/bison_parser.y
@@ -169,6 +169,7 @@
   hsql::RowLockWaitPolicy lock_wait_policy_t;
 
   hsql::ImportExportOptions* import_export_option_t;
+  std::pair<hsql::CsvOptionType, char*>* csv_option_t;
 
   // clang-format off
 }
@@ -199,6 +200,10 @@
   }
   delete ($$);
 } <table_vec> <table_element_vec> <update_vec> <expr_vec> <order_vec> <stmt_vec>
+%destructor {
+  free($$->second);
+  delete ($$);
+} <csv_option_t>
 %destructor { delete ($$); } <*>
 
 
@@ -294,6 +299,7 @@
 // ImportType is used for compatibility reasons
 %type <import_type_t>          file_type
 %type <import_export_option_t> opt_import_export_options import_export_options
+%type <csv_option_t>           csv_option
 
 %type <str_vec>                ident_commalist opt_column_list
 %type <expr_vec>               expr_list select_list opt_extended_literal_list extended_literal_list hint_list opt_hints opt_partition
@@ -469,6 +475,10 @@ import_statement : IMPORT FROM file_type FILE file_path INTO table_name {
     $$->encoding = $5->encoding;
     $5->encoding = nullptr;
   }
+  if ($5->csv_options) {
+    $$->csv_options = $5->csv_options;
+    $5->csv_options = nullptr;
+  }
   delete $5;
 };
 
@@ -499,6 +509,11 @@ import_export_options : import_export_options ',' FORMAT file_type {
     yyerror(&yyloc, result, scanner, "File type must only be provided once.");
     YYERROR;
   }
+  if ($1->csv_options && $4 != kImportCSV && $4 != kImportAuto) {
+    delete $1;
+    yyerror(&yyloc, result, scanner, "CSV options (DELIMITER, NULL, QUOTE) are only allowed for CSV files.");
+    YYERROR;
+  }
   $1->format = $4;
   $$ = $1;
 }
@@ -519,7 +534,53 @@ import_export_options : import_export_options ',' FORMAT file_type {
 | ENCODING STRING {
   $$ = new ImportExportOptions{};
   $$->encoding = $2;
-};
+}
+| import_export_options ',' csv_option {
+  if ($1->format != kImportAuto && $1->format != kImportCSV) {
+    delete $1;
+    free($3->second);
+    delete $3;
+    yyerror(&yyloc, result, scanner, "CSV options (DELIMITER, NULL, QUOTE) are only allowed for CSV files.");
+    YYERROR;
+  }
+
+  if ($1->csv_options == nullptr) {
+    $1->csv_options = new CsvOptions{};
+  }
+
+  if (!$1->csv_options->accept_csv_option($3)) {
+    free($3->second);
+    delete $3;
+    delete $1;
+    yyerror(&yyloc, result, scanner, "CSV options (DELIMITER, NULL, QUOTE) cannot be provided more than once.");
+    YYERROR;
+  }
+
+  delete $3;
+  $$ = $1;
+}
+| csv_option {
+  $$ = new ImportExportOptions{};
+  $$->csv_options = new CsvOptions{};
+  $$->csv_options->accept_csv_option($1);
+
+  delete $1;
+}
+
+csv_option : IDENTIFIER STRING {
+  if (strcasecmp($1, "DELIMITER") == 0) {
+    $$ = new std::pair<CsvOptionType, char*>(CsvOptionType::Delimiter, $2);
+  } else if (strcasecmp($1, "QUOTE") == 0) {
+    $$ = new std::pair<CsvOptionType, char*>(CsvOptionType::Quote, $2);
+  } else {
+    free($1);
+    free($2);
+    yyerror(&yyloc, result, scanner, "Unknown CSV option.");
+    YYERROR;
+  }
+  free($1);
+}
+| NULL STRING { $$ = new std::pair<CsvOptionType, char*>(CsvOptionType::Null, $2); }
 
 /******************************
  * Export Statement
@@ -535,6 +596,10 @@ export_statement : COPY table_name TO file_path opt_import_export_options {
     $$->encoding = $5->encoding;
     $5->encoding = nullptr;
   }
+  if ($5->csv_options) {
+    $$->csv_options = $5->csv_options;
+    $5->csv_options = nullptr;
+  }
   delete $5;
 }
 | COPY select_with_paren TO file_path opt_import_export_options {
@@ -545,6 +610,10 @@ export_statement : COPY table_name TO file_path opt_import_export_options {
     $$->encoding = $5->encoding;
     $5->encoding = nullptr;
   }
+  if ($5->csv_options) {
+    $$->csv_options = $5->csv_options;
+    $5->csv_options = nullptr;
+  }
   delete $5;
 };
 
diff --git a/src/sql/ExportStatement.h b/src/sql/ExportStatement.h
@@ -18,6 +18,7 @@ struct ExportStatement : SQLStatement {
   char* tableName;
   SelectStatement* select;
   char* encoding;
+  CsvOptions* csv_options;
 };
 
 }  // namespace hsql
diff --git a/src/sql/ImportExportOptions.h b/src/sql/ImportExportOptions.h
@@ -1,6 +1,8 @@
 #ifndef SQLPARSER_IMPORT_EXPORT_OPTIONS_H
 #define SQLPARSER_IMPORT_EXPORT_OPTIONS_H
 
+#include <utility>
+
 namespace hsql {
 
 // Name unchanged for compatibility. Historically, this was only used for import statements before we introduced export
@@ -13,12 +15,30 @@ enum ImportType {
   kImportAuto
 };
 
+enum CsvOptionType {
+  Delimiter,
+  Null,
+  Quote,
+};
+
+struct CsvOptions {
+  CsvOptions();
+  ~CsvOptions();
+
+  char* delimiter;
+  char* null;
+  char* quote;
+
+  bool accept_csv_option(std::pair<CsvOptionType, char*>* option);
+};
+
 struct ImportExportOptions {
   ImportExportOptions();
   ~ImportExportOptions();
 
   ImportType format;
   char* encoding;
+  CsvOptions* csv_options;
 };
 
 }  // namespace hsql
diff --git a/src/sql/ImportStatement.h b/src/sql/ImportStatement.h
@@ -17,6 +17,7 @@ struct ImportStatement : SQLStatement {
   char* tableName;
   Expr* whereClause;
   char* encoding;
+  CsvOptions* csv_options;
 };
 
 }  // namespace hsql
diff --git a/src/sql/statements.cpp b/src/sql/statements.cpp
@@ -140,19 +140,56 @@ ExportStatement::ExportStatement(ImportType type)
       schema(nullptr),
       tableName(nullptr),
       select(nullptr),
-      encoding(nullptr) {}
+      encoding(nullptr),
+      csv_options(nullptr) {}
 
 ExportStatement::~ExportStatement() {
   free(filePath);
   free(schema);
   free(tableName);
   delete select;
   free(encoding);
+  delete csv_options;
 }
 
-ImportExportOptions::ImportExportOptions() : format(kImportAuto), encoding(nullptr) {}
+CsvOptions::CsvOptions() : delimiter(nullptr), null(nullptr), quote(nullptr) {}
+CsvOptions::~CsvOptions() {
+  free(delimiter);
+  free(null);
+  free(quote);
+}
+
+bool CsvOptions::accept_csv_option(std::pair<CsvOptionType, char*>* option) {
+  switch (option->first) {
+    case CsvOptionType::Delimiter:
+      if (delimiter != nullptr) {
+        return false;
+      }
+      delimiter = option->second;
+      break;
+    case CsvOptionType::Null:
+      if (null != nullptr) {
+        return false;
+      }
+      null = option->second;
+      break;
+    case CsvOptionType::Quote:
+      if (quote != nullptr) {
+        return false;
+      }
+      quote = option->second;
+      break;
+  }
+
+  return true;
+}
 
-ImportExportOptions::~ImportExportOptions() { free(encoding); }
+ImportExportOptions::ImportExportOptions() : format(kImportAuto), encoding(nullptr), csv_options(nullptr) {}
+
+ImportExportOptions::~ImportExportOptions() {
+  free(encoding);
+  delete csv_options;
+}
 
 // ImportStatement
 ImportStatement::ImportStatement(ImportType type)
@@ -162,14 +199,16 @@ ImportStatement::ImportStatement(ImportType type)
       schema(nullptr),
       tableName(nullptr),
       whereClause(nullptr),
-      encoding(nullptr) {}
+      encoding(nullptr),
+      csv_options(nullptr) {}
 
 ImportStatement::~ImportStatement() {
   free(filePath);
   free(schema);
   free(tableName);
   delete whereClause;
   free(encoding);
+  delete csv_options;
 }
 
 // InsertStatement
diff --git a/test/queries/queries-bad.sql b/test/queries/queries-bad.sql
@@ -102,3 +102,11 @@
 !SELECT * FROM students ORDER BY name FIRST;
 !SELECT * FROM students ORDER BY name ASC LAST;
 !SELECT * FROM students ORDER BY name DESC NULLS gibberish;
+# CSV options
+!COPY students FROM 'file_path' WITH (FORMAT TBL, DELIMITER '|', NULL '', QUOTE '"');
+!COPY students FROM 'file_path' WITH (DELIMITER '|', NULL '', QUOTE '"', FORMAT TBL);
+!COPY students FROM 'file_path' WITH (DELIMITER '|', NULL '', FORMAT TBL, QUOTE '"');
+!COPY students FROM 'file_path' WITH (DELIMITER '|', NULL '', QUOTE '"', NULL 'a');
+!COPY students FROM 'file_path' WITH (NULL '', QUOTE '"', DELIMITER '|', DELIMITER '/');
+!COPY students FROM 'file_path' WITH (QUOTE '"', NULL '', DELIMITER '/', QUOTE '_',);
+!COPY students FROM 'file_path' WITH (FORMAT CSV, QUOTE '"', DELIMINIMITER '|');
diff --git a/test/queries/queries-good.sql b/test/queries/queries-good.sql
@@ -74,6 +74,10 @@ COPY students FROM 'file_path' WITH (FORMAT TBL);
 COPY students FROM 'file_path' WITH (FORMAT CSV);
 COPY students FROM 'file_path' WITH (FORMAT BIN);
 COPY students FROM 'file_path' WITH (FORMAT BINARY);
+COPY students FROM 'file_path' WITH (FORMAT CSV, DELIMITER '|', NULL '', QUOTE '"');
+COPY students FROM 'file_path' WITH (DELIMITER '|', NULL '', FORMAT CSV, QUOTE '"');
+COPY students FROM 'file_path' WITH (DELIMITER '|', NULL '', QUOTE '"');
+COPY students FROM 'file_path' WITH (DELIMITER '|', FORMAT CSV);
 COPY students FROM 'file_path' (FORMAT TBL);
 COPY good_students FROM 'file_path' WHERE grade > (SELECT AVG(grade) from alumni);
 COPY students TO 'student.tbl';
diff --git a/test/sql_tests.cpp b/test/sql_tests.cpp
@@ -466,16 +466,23 @@ TEST(ImportStatementTest) {
 }
 
 TEST(CopyStatementTest) {
-  TEST_PARSE_SINGLE_SQL("COPY students FROM 'students_file' WITH (FORMAT BINARY);", kStmtImport, ImportStatement,
-                        import_result, import_stmt);
+  TEST_PARSE_SINGLE_SQL("COPY students FROM 'students_file' WITH (FORMAT CSV, DELIMITER '|', NULL '', QUOTE '\"');",
+                        kStmtImport, ImportStatement, import_result, import_stmt);
 
-  ASSERT_EQ(import_stmt->type, kImportBinary);
+  ASSERT_EQ(import_stmt->type, kImportCSV);
   ASSERT_NOTNULL(import_stmt->tableName);
   ASSERT_STREQ(import_stmt->tableName, "students");
   ASSERT_NOTNULL(import_stmt->filePath);
   ASSERT_STREQ(import_stmt->filePath, "students_file");
   ASSERT_NULL(import_stmt->whereClause);
   ASSERT_NULL(import_stmt->encoding);
+  ASSERT_NOTNULL(import_stmt->csv_options);
+  ASSERT_NOTNULL(import_stmt->csv_options->delimiter);
+  ASSERT_STREQ(import_stmt->csv_options->delimiter, "|");
+  ASSERT_NOTNULL(import_stmt->csv_options->null);
+  ASSERT_STREQ(import_stmt->csv_options->null, "");
+  ASSERT_NOTNULL(import_stmt->csv_options->quote);
+  ASSERT_STREQ(import_stmt->csv_options->quote, "\"");
 
   TEST_PARSE_SINGLE_SQL("COPY students FROM 'students_file' WHERE lastname = 'Potter';", kStmtImport, ImportStatement,
                         import_filter_result, import_filter_stmt);
@@ -492,17 +499,19 @@ TEST(CopyStatementTest) {
   ASSERT_EQ(import_filter_stmt->whereClause->expr2->type, kExprLiteralString);
   ASSERT_STREQ(import_filter_stmt->whereClause->expr2->name, "Potter");
   ASSERT_NULL(import_filter_stmt->encoding);
+  ASSERT_NULL(import_filter_stmt->csv_options);
 
-  TEST_PARSE_SINGLE_SQL("COPY students TO 'students_file' WITH (ENCODING 'FSST', FORMAT CSV);", kStmtExport,
+  TEST_PARSE_SINGLE_SQL("COPY students TO 'students_file' WITH (ENCODING 'FSST', FORMAT BINARY);", kStmtExport,
                         ExportStatement, export_table_result, export_table_stmt);
 
-  ASSERT_EQ(export_table_stmt->type, kImportCSV);
+  ASSERT_EQ(export_table_stmt->type, kImportBinary);
   ASSERT_NOTNULL(export_table_stmt->tableName);
   ASSERT_STREQ(export_table_stmt->tableName, "students");
   ASSERT_NOTNULL(export_table_stmt->filePath);
   ASSERT_STREQ(export_table_stmt->filePath, "students_file");
   ASSERT_NULL(export_table_stmt->select);
   ASSERT_STREQ(export_table_stmt->encoding, "FSST");
+  ASSERT_NULL(export_table_stmt->csv_options);
 
   TEST_PARSE_SINGLE_SQL(
       "COPY (SELECT firstname, lastname FROM students) TO 'students_file' WITH (ENCODING 'Dictionary');", kStmtExport,
@@ -513,6 +522,7 @@ TEST(CopyStatementTest) {
   ASSERT_NOTNULL(export_select_stmt->filePath);
   ASSERT_STREQ(export_select_stmt->filePath, "students_file");
   ASSERT_STREQ(export_select_stmt->encoding, "Dictionary");
+  ASSERT_NULL(export_select_stmt->csv_options);
 
   ASSERT_NOTNULL(export_select_stmt->select);
   const auto& select_stmt = export_select_stmt->select;
