Load and export database as JSON-formatted buffer to enable database encryption (#216)

* Add `import_from_buffer()`- and `export_to_buffer()`-methods to export/import a database from/to a PackedByteArray.
* Updated relevant documentation
* Added example of AES database encryption to demo

Author: versjon

README.md

@@ -215,6 +215,18 @@ Additionally, a video tutorial by [Mitch McCollum (finepointcgi)](https://github
 
     Exports the database structure and content to `export_path.json` as a backup or for ease of editing.
 
+- Boolean success = **import_from_buffer(** PackedByteArray input_buffer **)**
+
+    Drops all database tables and imports the database structure and content encoded in JSON-formatted input_buffer.
+
+    Can be used together with `export_to_buffer()` to implement database encryption.
+
+- PackedByteArray output_buffer = **export_to_buffer()**
+
+    Returns the database structure and content as JSON-formatted buffer. 
+
+    Can be used together with `import_from_buffer()` to implement database encryption.
+
 - Boolean success = **create_function(** String function_name, FuncRef function_reference, int number_of_arguments **)**
 
     Bind a [scalar SQL function](https://www.sqlite.org/appfunc.html) to the database that can then be used in subsequent queries.

demo/database.gd

@@ -40,6 +40,7 @@ func _ready():
 	example_of_read_only_database()
 	example_of_database_persistency()
 	example_of_fts5_usage()
+	example_of_encrypted_database()
 
 func cprint(text : String) -> void:
 	print(text)
@@ -174,7 +175,7 @@ func example_of_basic_database_querying():
 
 	# Close the current database
 	db.close_db()
-	
+
 	# Import (and, consequently, open) a database from an old backup json-file
 	cprint("Overwriting database content with old backup...")
 	db.import_from_json(json_name + "_old")
@@ -494,3 +495,71 @@ func example_of_fts5_usage():
 
 	# Close the current database
 	db.close_db()
+
+
+# Example of "secure" database storage using AES encryption
+# Mind that this is still vulnerable to various attacks like memory dumps during runtime
+func example_of_encrypted_database():
+	# Unlike in this example, the key should never be stored persistently inside the project
+	var key = "secret_key123456" # Key must be either 16 or 32 bytes
+
+	# Create a table containing "sensitive" data
+	var table_dict : Dictionary = Dictionary()
+	table_dict["name"] = {"data_type":"text", "not_null": true}
+
+	db = SQLite.new()
+	# Use in-memory shared database to avoid storing database on disk
+	db.path = "file::memory:?cache=shared"
+	db.verbosity_level = verbosity_level
+	db.open_db()
+	db.create_table("agents", table_dict)
+
+	var row_array : Array = []
+	var row_dict : Dictionary = Dictionary()
+	for i in range(0,names.size()):
+		row_dict["name"] = names[i]
+		row_array.append(row_dict.duplicate())
+
+		db.insert_row("agents", row_dict)
+		row_dict.clear()
+
+	# Output original database
+	var agents_pre: Array = db.select_rows("agents", "", ["name"])
+	var names_pre = agents_pre.map(func(agent): return agent["name"])
+	cprint("Agent names pre encryption: " + str(names_pre))
+
+	# Export database as JSON-formatted buffer
+	var unencrypted: PackedByteArray = db.export_to_buffer()
+	db.drop_table("agents")
+
+	# Add padding
+	var buffer_size = unencrypted.size()
+	var padding_size = 16-posmod(buffer_size, 16)
+	var padding := PackedByteArray()
+	padding.resize(padding_size)
+	padding.fill(padding_size)
+	unencrypted.append_array(padding)
+
+	var aes = AESContext.new()
+	aes.start(AESContext.MODE_ECB_ENCRYPT, key.to_utf8_buffer())
+	var encrypted: PackedByteArray = aes.update(unencrypted)
+	aes.finish()
+	# encrypted now contains the padded AES-encrypted database and can be stored using FileAccess
+
+	aes.start(AESContext.MODE_ECB_DECRYPT, key.to_utf8_buffer())
+	var decrypted: PackedByteArray = aes.update(encrypted)
+	aes.finish()
+	# decrypted now contains the padded AES-encrypted database
+
+	# Remove padding
+	buffer_size = decrypted.size()
+	padding_size = decrypted.get(buffer_size-1)
+	decrypted = decrypted.slice(0, buffer_size-padding_size)
+
+	# Import database from JSON-formatted buffer
+	db.import_from_buffer(decrypted)
+
+	# Output database after encryption and decryption
+	var agents_post: Array = db.select_rows("agents", "", ["name"])
+	var names_post = agents_post.map(func(agent): return agent["name"])
+	cprint("Agent names post decryption: " + str(names_post))

doc_classes/SQLite.xml

@@ -162,6 +162,20 @@
 				Exports the database structure and content to [code]export_path.json[/code] as a backup or for ease of editing.
 			</description>
 		</method>
+		<method name="import_from_buffer">
+			<return type="bool" />
+			<description>
+				Drops all database tables and imports the database structure and content encoded in JSON-formatted input buffer.
+				Can be used together with [method SQLite.export_to_buffer] to implement database encryption.
+			</description>
+		</method>
+		<method name="export_to_buffer">
+			<return type="PackedByteArray" />
+			<description>
+				Returns the database structure and content as JSON-formatted buffer.
+				Can be used together with [method SQLite.import_from_buffer] to implement database encryption.
+			</description>
+		</method>
 		<method name="create_function">
 			<return type="bool" />
 			<description>

src/gdsqlite.cpp

@@ -26,6 +26,8 @@ void SQLite::_bind_methods() {
 
 	ClassDB::bind_method(D_METHOD("import_from_json", "import_path"), &SQLite::import_from_json);
 	ClassDB::bind_method(D_METHOD("export_to_json", "export_path"), &SQLite::export_to_json);
+	ClassDB::bind_method(D_METHOD("import_from_buffer", "json_buffer"), &SQLite::import_from_buffer);
+	ClassDB::bind_method(D_METHOD("export_to_buffer"), &SQLite::export_to_buffer);
 
 	ClassDB::bind_method(D_METHOD("get_autocommit"), &SQLite::get_autocommit);
 	ClassDB::bind_method(D_METHOD("compileoption_used", "option_name"), &SQLite::compileoption_used);
@@ -819,15 +821,53 @@ bool SQLite::import_from_json(String import_path) {
 		ERR_PRINT("GDSQLite Error: " + String(std::strerror(errno)) + " (" + import_path + ")");
 		return false;
 	}
+
 	std::stringstream buffer;
 	buffer << ifs.rdbuf();
 	std::string str = buffer.str();
-	String json_string = String::utf8(str.c_str());
 	ifs.close();
 
-	/* Attempt to parse the result and, if unsuccessful, throw a parse error specifying the erroneous line */
+	String json_string = String::utf8(str.c_str());
+	PackedByteArray json_buffer = json_string.to_utf8_buffer();
+
+	return import_from_buffer(json_buffer);
+}
+
+bool SQLite::export_to_json(String export_path) {
+	PackedByteArray json_buffer = export_to_buffer();
+	String json_string = json_buffer.get_string_from_utf8();
+
+	/* Add .json to the import_path String if not present */
+	String ending = String(".json");
+	if (!export_path.ends_with(ending)) {
+		export_path += ending;
+	}
+	/* Find the real path */
+	export_path = ProjectSettings::get_singleton()->globalize_path(export_path.strip_edges());
+	CharString dummy_path = export_path.utf8();
+	const char *char_path = dummy_path.get_data();
+	//const char *char_path = export_path.alloc_c_string();
+
+	std::ofstream ofs(char_path, std::ios::trunc);
+	if (ofs.fail()) {
+		ERR_PRINT("GDSQLite Error: " + String(std::strerror(errno)) + " (" + export_path + ")");
+		return false;
+	}
+
+	CharString dummy_string = json_string.utf8();
+	ofs << dummy_string.get_data();
+	//ofs << json_string.alloc_c_string();
+	ofs.close();
+
+	return true;
+}
+
+bool SQLite::import_from_buffer(PackedByteArray json_buffer) {
+	/* Attempt to parse the input json_string and, if unsuccessful, throw a parse error specifying the erroneous line */
+
 	Ref<JSON> json;
 	json.instantiate();
+	String json_string = json_buffer.get_string_from_utf8();
 	Error error = json->parse(json_string);
 	if (error != Error::OK) {
 		/* Throw a parsing error */
@@ -934,7 +974,7 @@ bool SQLite::import_from_json(String import_path) {
 	return true;
 }
 
-bool SQLite::export_to_json(String export_path) {
+PackedByteArray SQLite::export_to_buffer() {
 	/* Get all names and sql templates for all tables present in the database */
 	query(String("SELECT name,sql,type FROM sqlite_master;"));
 	TypedArray<Dictionary> database_array = query_result.duplicate(true);
@@ -943,7 +983,7 @@ bool SQLite::export_to_json(String export_path) {
 	remove_shadow_tables(database_array);
 #endif
 	int64_t number_of_objects = database_array.size();
-	/* Construct a Dictionary for each table, convert it to JSON and write it to file */
+	/* Construct a Dictionary for each table, convert it to JSON and write it to String */
 	for (int64_t i = 0; i <= number_of_objects - 1; i++) {
 		Dictionary object_dict = database_array[i];
 
@@ -989,31 +1029,11 @@ bool SQLite::export_to_json(String export_path) {
 		}
 	}
 
-	/* Add .json to the import_path String if not present */
-	String ending = String(".json");
-	if (!export_path.ends_with(ending)) {
-		export_path += ending;
-	}
-	/* Find the real path */
-	export_path = ProjectSettings::get_singleton()->globalize_path(export_path.strip_edges());
-	CharString dummy_path = export_path.utf8();
-	const char *char_path = dummy_path.get_data();
-	//const char *char_path = export_path.alloc_c_string();
-
-	std::ofstream ofs(char_path, std::ios::trunc);
-	if (ofs.fail()) {
-		ERR_PRINT("GDSQLite Error: " + String(std::strerror(errno)) + " (" + export_path + ")");
-		return false;
-	}
 	Ref<JSON> json;
 	json.instantiate();
 	String json_string = json->stringify(database_array, "\t");
-	CharString dummy_string = json_string.utf8();
-	ofs << dummy_string.get_data();
-	//ofs << json_string.alloc_c_string();
-	ofs.close();
-
-	return true;
+	PackedByteArray json_buffer = json_string.to_utf8_buffer();
+	return json_buffer;
 }
 
 bool SQLite::validate_json(const Array &database_array, std::vector<object_struct> &objects_to_import) {

src/gdsqlite.h

@@ -91,6 +91,9 @@ class SQLite : public RefCounted {
 	bool import_from_json(String import_path);
 	bool export_to_json(String export_path);
 
+	bool import_from_buffer(PackedByteArray json_buffer);
+	PackedByteArray export_to_buffer();
+
 	int get_autocommit() const;
 	int compileoption_used(const String &option_name) const;