Skip to content

Commit 889a806

Browse files
illwieckzillwieckz
committed
FileSystem: implement internal paks
1 parent 5c7bc3f commit 889a806

File tree

4 files changed

+152
-21
lines changed

4 files changed

+152
-21
lines changed

‎cmake/DaemonSourceGenerator.cmake‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,7 @@ macro(daemon_embed_files basename dir list format targetname)
103103

104104
set(embed_header_file "${DAEMON_GENERATED_DIR}/${DAEMON_EMBEDDED_FILES_HEADER}")
105105
file(GENERATE OUTPUT "${embed_header_file}" CONTENT "${embed_header_text}")
106+
add_definitions(-DDAEMON_EMBEDDED_FILES_HEADER="${DAEMON_EMBEDDED_FILES_HEADER}")
106107
endif()
107108

108109
string(APPEND embed_CPP_text

‎src/common/FileSystem.cpp‎

Lines changed: 136 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,35 @@ static Cvar::Cvar<bool> fs_legacypaks("fs_legacypaks", "also load pk3s, ignoring
129129
static Cvar::Cvar<int> fs_maxSymlinkDepth("fs_maxSymlinkDepth", "max depth of symlinks in zip paks (0 means disabled)", Cvar::NONE, 1);
130130
static Cvar::Cvar<std::string> fs_pakprefixes("fs_pakprefixes", "prefixes to look for paks to load", 0, "");
131131

132+
builtinPakMap_t builtinPakMap = {};
133+
134+
std::vector<PakInfo> builtinPaks = {};
135+
136+
void ClearBuiltinPaks()
137+
{
138+
builtinPakMap.clear();
139+
140+
builtinPaks.clear();
141+
builtinPaks.shrink_to_fit();
142+
}
143+
144+
void AddBuiltinPak(const std::string& name, const std::string& version, const embeddedFileMap_t& map)
145+
{
146+
PakInfo *pak = new PakInfo();
147+
pak->name = Str::Format( "*%s", name );
148+
pak->version = version;
149+
pak->checksum = 0;
150+
pak->type = pakType_t::PAK_ZIP;
151+
pak->path = Str::Format( "*%s_%s", name, version );
152+
153+
builtinPaks.push_back(*pak);
154+
155+
Log::Debug("Adding builtin pak with %d files: %s", map.size(), pak->name);
156+
builtinPakMap[pak->name] = map;
157+
158+
PakPath::LoadPak(*pak);
159+
}
160+
132161
bool UseLegacyPaks()
133162
{
134163
return fs_legacypaks.Get();
@@ -1257,8 +1286,13 @@ static void InternalLoadPak(
12571286
bool hasDeps = false;
12581287
offset_t depsOffset = 0;
12591288
ZipArchive zipFile;
1289+
12601290
bool isLegacy = pak.version.empty();
12611291

1292+
bool isBuiltin = pak.type == pakType_t::PAK_ZIP && pak.name[ 0 ] == '*';
1293+
bool isDir = pak.type == pakType_t::PAK_DIR;
1294+
bool isZip = pak.type == pakType_t::PAK_ZIP && !isBuiltin;
1295+
12621296
// Check if this pak has already been loaded to avoid recursive dependencies
12631297
for (auto& x: loadedPaks) {
12641298
// If the prefix is a superset of our current prefix, then it already
@@ -1267,13 +1301,15 @@ static void InternalLoadPak(
12671301
return;
12681302
}
12691303

1270-
if (pak.type == pakType_t::PAK_ZIP) {
1304+
if (isBuiltin) {
1305+
fsLogs.WithoutSuppression().Notice("Loading builtin pak '%s'...", pak.path.c_str());
1306+
} else if (isZip) {
12711307
if (!isLegacy) {
12721308
fsLogs.WithoutSuppression().Notice("Loading pak '%s'...", pak.path.c_str());
12731309
} else {
12741310
fsLogs.WithoutSuppression().Notice("Loading legacy pak '%s'...", pak.path.c_str());
12751311
}
1276-
} else if (pak.type == pakType_t::PAK_DIR) {
1312+
} else if (isDir) {
12771313
if (!isLegacy) {
12781314
fsLogs.WithoutSuppression().Notice("Loading pakdir '%s'...", pak.path.c_str());
12791315
} else {
@@ -1292,7 +1328,14 @@ static void InternalLoadPak(
12921328
loadedPak.path = pak.path;
12931329

12941330
// Update the list of files, but don't overwrite existing files, so the sort order is preserved
1295-
if (pak.type == pakType_t::PAK_DIR) {
1331+
if (isBuiltin) {
1332+
loadedPak.fd = -1;
1333+
for (auto& it : builtinPakMap[pak.name])
1334+
{
1335+
Log::Debug("Adding file from builtin pak %s: %s", pak.name, it.first);
1336+
fileMap.emplace(it.first, std::pair<uint32_t, offset_t>(loadedPaks.size() - 1, 0));
1337+
}
1338+
} else if (isDir) {
12961339
loadedPak.fd = -1;
12971340
auto dirRange = RawPath::ListFilesRecursive(pak.path, err);
12981341
if (err)
@@ -1316,7 +1359,7 @@ static void InternalLoadPak(
13161359
if (err)
13171360
return;
13181361
}
1319-
} else if (pak.type == pakType_t::PAK_ZIP) {
1362+
} else if (isZip) {
13201363
// Open file
13211364
loadedPak.fd = my_open(pak.path, openMode_t::MODE_READ);
13221365
if (loadedPak.fd == -1) {
@@ -1380,16 +1423,16 @@ static void InternalLoadPak(
13801423
// Directories (aka a dpkdir) don't need timestamp.
13811424
// Fixes Windows bug where calling _wstat64i with trailing slash causes "file not found" error.
13821425
// For future stat calls on directories, trim the trailing slash (if exists)
1383-
if (pak.type == pakType_t::PAK_ZIP) {
1426+
if (isZip) {
13841427
loadedPak.timestamp = FS::RawPath::FileTimestamp(pak.path, err);
13851428
if (err)
13861429
return;
13871430
}
13881431

13891432
loadedPak.pathPrefix = pathPrefix;
13901433

1391-
// Legacy paks don't have version neither checksum
1392-
if (!isLegacy) {
1434+
// Legacy and builtin paks don't have version neither checksum
1435+
if (!isLegacy && !isBuiltin) {
13931436
// If an explicit checksum was requested, verify that the pak we loaded is the one we are expecting
13941437
if (expectedChecksum && realChecksum != *expectedChecksum) {
13951438
SetErrorCodeFilesystem(err, filesystem_error::wrong_pak_checksum, pak.path);
@@ -1406,14 +1449,16 @@ static void InternalLoadPak(
14061449
if (!isLegacy) {
14071450
if (hasDeleted) {
14081451
std::string deletedData;
1409-
if (pak.type == pakType_t::PAK_DIR) {
1452+
if (isBuiltin) {
1453+
// Not implemented.
1454+
} else if (isDir) {
14101455
File depsFile = RawPath::OpenRead(Path::Build(pak.path, PAK_DELETED_FILE), err);
14111456
if (err)
14121457
return;
14131458
deletedData = depsFile.ReadAll(err);
14141459
if (err)
14151460
return;
1416-
} else if (pak.type == pakType_t::PAK_ZIP) {
1461+
} else if (isZip) {
14171462
zipFile.OpenFile(deletedOffset, err);
14181463
if (err)
14191464
return;
@@ -1434,14 +1479,16 @@ static void InternalLoadPak(
14341479
// Load dependencies (non-legacy paks (pk3) only)
14351480
if (loadDeps && hasDeps) {
14361481
std::string depsData;
1437-
if (pak.type == pakType_t::PAK_DIR) {
1482+
if (isBuiltin) {
1483+
// Not implemented.
1484+
} else if (isDir) {
14381485
File depsFile = RawPath::OpenRead(Path::Build(pak.path, PAK_DEPS_FILE), err);
14391486
if (err)
14401487
return;
14411488
depsData = depsFile.ReadAll(err);
14421489
if (err)
14431490
return;
1444-
} else if (pak.type == pakType_t::PAK_ZIP) {
1491+
} else if (isZip) {
14451492
zipFile.OpenFile(depsOffset, err);
14461493
if (err)
14471494
return;
@@ -1491,6 +1538,7 @@ void ClearPaks()
14911538
close(x.fd);
14921539
}
14931540
loadedPaks.clear();
1541+
FS::ClearBuiltinPaks();
14941542
FS::RefreshPaks();
14951543
}
14961544
#else // BUILD_VM
@@ -1549,7 +1597,18 @@ std::string ReadFile(Str::StringRef path, std::error_code& err)
15491597
}
15501598

15511599
const LoadedPakInfo& pak = loadedPaks[it->second.first];
1552-
if (pak.type == pakType_t::PAK_DIR) {
1600+
1601+
bool isBuiltin = pak.type == pakType_t::PAK_ZIP && pak.name[0] == '*';
1602+
bool isDir = pak.type == pakType_t::PAK_DIR;
1603+
bool isZip = pak.type == pakType_t::PAK_ZIP && !isBuiltin;
1604+
1605+
if (isBuiltin) {
1606+
const embeddedFileMapEntry_t& entry = builtinPakMap[pak.name][path];
1607+
std::string out;
1608+
out.resize(entry.size);
1609+
memcpy(&out[0], entry.data, entry.size);
1610+
return out;
1611+
} else if (isDir) {
15531612
// Open file
15541613
File file = RawPath::OpenRead(Path::Build(pak.path, it->first), err);
15551614
if (err)
@@ -1565,7 +1624,7 @@ std::string ReadFile(Str::StringRef path, std::error_code& err)
15651624
out.resize(length);
15661625
file.Read(&out[0], length, err);
15671626
return out;
1568-
} else if (pak.type == pakType_t::PAK_ZIP) {
1627+
} else if (isZip) {
15691628
// Open zip
15701629
ZipArchive zipFile = ZipArchive::Open(pak.fd, err);
15711630
if (err)
@@ -1604,12 +1663,19 @@ void CopyFile(Str::StringRef path, const File& dest, std::error_code& err)
16041663
}
16051664

16061665
const LoadedPakInfo& pak = loadedPaks[it->second.first];
1607-
if (pak.type == pakType_t::PAK_DIR) {
1666+
1667+
bool isBuiltin = pak.type == pakType_t::PAK_ZIP && pak.name[0] == '*';
1668+
bool isDir = pak.type == pakType_t::PAK_DIR;
1669+
bool isZip = pak.type == pakType_t::PAK_ZIP && !isBuiltin;
1670+
1671+
if (isBuiltin) {
1672+
// Not implemented
1673+
} else if (isDir) {
16081674
File file = RawPath::OpenRead(Path::Build(pak.path, it->first), err);
16091675
if (err)
16101676
return;
16111677
file.CopyTo(dest, err);
1612-
} else if (pak.type == pakType_t::PAK_ZIP) {
1678+
} else if (isZip) {
16131679
// Open zip
16141680
ZipArchive zipFile = ZipArchive::Open(pak.fd, err);
16151681
if (err)
@@ -1672,7 +1738,15 @@ std::chrono::system_clock::time_point FileTimestamp(Str::StringRef path, std::er
16721738
}
16731739

16741740
const LoadedPakInfo& pak = loadedPaks[it->second.first];
1675-
if (pak.type == pakType_t::PAK_DIR) {
1741+
1742+
bool isBuiltin = pak.type == pakType_t::PAK_ZIP && pak.name[0] == '*';
1743+
bool isDir = pak.type == pakType_t::PAK_DIR;
1744+
bool isZip = pak.type == pakType_t::PAK_ZIP && !isBuiltin;
1745+
1746+
if (isBuiltin) {
1747+
// Not implemented.
1748+
return {};
1749+
} else if (isDir) {
16761750
#ifdef BUILD_VM
16771751
Util::optional<uint64_t> result;
16781752
VM::SendMsg<VM::FSPakPathTimestampMsg>(it->second.first, it->first, result);
@@ -1686,7 +1760,7 @@ std::chrono::system_clock::time_point FileTimestamp(Str::StringRef path, std::er
16861760
#else
16871761
return RawPath::FileTimestamp(Path::Build(pak.path, it->first), err);
16881762
#endif
1689-
} else if (pak.type == pakType_t::PAK_ZIP) {
1763+
} else if (isZip) {
16901764
return pak.timestamp;
16911765
}
16921766

@@ -2636,6 +2710,10 @@ void RefreshPaks()
26362710
if (a.checksum != b.checksum)
26372711
return a.checksum > b.checksum;
26382712

2713+
// Prefer builtin packages to zip packages.
2714+
if (b.type == pakType_t::PAK_ZIP && a.type == pakType_t::PAK_ZIP && b.name[0] == '*')
2715+
return true;
2716+
26392717
// Prefer zip packages to directory packages
26402718
if (b.type == pakType_t::PAK_ZIP && a.type != pakType_t::PAK_ZIP)
26412719
return true;
@@ -2660,6 +2738,16 @@ static const PakInfo* FindPakNoPrefix(Str::StringRef name)
26602738

26612739
const PakInfo* FindPak(Str::StringRef name)
26622740
{
2741+
#if defined(BUILD_ENGINE)
2742+
for ( auto& builtinPak : builtinPaks)
2743+
{
2744+
if ( builtinPak.name == name )
2745+
{
2746+
return &builtinPak;
2747+
}
2748+
}
2749+
#endif
2750+
26632751
Cmd::Args pakprefixes(Cvar::GetValue("fs_pakprefixes"));
26642752
for (const std::string &pakprefix: pakprefixes)
26652753
{
@@ -2715,7 +2803,12 @@ const PakInfo* FindPak(Str::StringRef name, Str::StringRef version, uint32_t che
27152803
return checksum > pakInfo.checksum;
27162804
});
27172805

2718-
if (iter == availablePaks.begin() || (iter - 1)->name != name || (iter - 1)->version != version || !(iter - 1)->checksum || *(iter - 1)->checksum != checksum) {
2806+
if (iter == availablePaks.begin()
2807+
|| (iter - 1)->name != name
2808+
|| (iter - 1)->version != version
2809+
|| !(iter - 1)->checksum
2810+
|| *(iter - 1)->checksum != checksum) {
2811+
27192812
// Try again, but this time look for the pak without a checksum. We will verify the checksum later.
27202813
iter = std::upper_bound(availablePaks.begin(), availablePaks.end(), name, [version](Str::StringRef name1, const PakInfo& pakInfo) -> bool {
27212814
int result = name1.compare(pakInfo.name);
@@ -2728,7 +2821,23 @@ const PakInfo* FindPak(Str::StringRef name, Str::StringRef version, uint32_t che
27282821
});
27292822

27302823
// Only allow zip packages because directories don't have a checksum
2731-
if (iter == availablePaks.begin() || (iter - 1)->type == pakType_t::PAK_DIR || (iter - 1)->name != name || (iter - 1)->version != version || (iter - 1)->checksum)
2824+
if (iter == availablePaks.begin())
2825+
return nullptr;
2826+
2827+
PakInfo& pak = (iter - 1)[0];
2828+
2829+
bool isBuiltin = pak.type == pakType_t::PAK_ZIP && pak.name[ 0 ] == '*';
2830+
bool isDir = pak.type == pakType_t::PAK_DIR;
2831+
2832+
if (isBuiltin)
2833+
return nullptr;
2834+
if (isDir)
2835+
return nullptr;
2836+
if (pak.name != name)
2837+
return nullptr;
2838+
if (pak.version != version)
2839+
return nullptr;
2840+
if (pak.checksum)
27322841
return nullptr;
27332842
}
27342843

@@ -2944,12 +3053,19 @@ void HandleFileSystemSyscall(int minor, Util::Reader& reader, IPC::Channel& chan
29443053
case VM::FS_PAKPATH_TIMESTAMP:
29453054
IPC::HandleMsg<VM::FSPakPathTimestampMsg>(channel, std::move(reader), [](uint32_t pakIndex, std::string path, Util::optional<uint64_t>& out) {
29463055
auto& loadedPaks = FS::PakPath::GetLoadedPaks();
3056+
29473057
if (loadedPaks.size() <= pakIndex)
29483058
return;
2949-
if (loadedPaks[pakIndex].type == pakType_t::PAK_ZIP)
3059+
3060+
const LoadedPakInfo& pak = loadedPaks[pakIndex];
3061+
bool isDir = pak.type == pakType_t::PAK_DIR;
3062+
3063+
if (!isDir)
29503064
return;
3065+
29513066
if (!Path::IsValid(path, false))
29523067
return;
3068+
29533069
std::error_code err;
29543070
std::chrono::system_clock::time_point t = RawPath::FileTimestamp(Path::Build(loadedPaks[pakIndex].path, path), err);
29553071
if (!err)

‎src/common/FileSystem.h‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,10 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
3737
#include "IPC/Channel.h"
3838
#endif
3939

40+
#if defined(BUILD_ENGINE)
41+
#include DAEMON_EMBEDDED_FILES_HEADER
42+
#endif
43+
4044
namespace FS {
4145

4246
bool UseLegacyPaks();
@@ -265,7 +269,7 @@ template<typename State> class DirectoryIterator {
265269
// Type of pak
266270
enum class pakType_t {
267271
PAK_ZIP, // Zip archive
268-
PAK_DIR // Directory
272+
PAK_DIR, // Directory
269273
};
270274

271275
// Information about a package
@@ -308,6 +312,12 @@ struct LoadedPakInfo: public PakInfo {
308312
std::string pathPrefix;
309313
};
310314

315+
#if defined(BUILD_ENGINE)
316+
using builtinPakMap_t = std::unordered_map<std::string, embeddedFileMap_t>;
317+
318+
void AddBuiltinPak(const std::string& name, const std::string& version, const embeddedFileMap_t& map);
319+
#endif
320+
311321
// Operations which work on files that are in packages. Packages should be used
312322
// for read-only assets which can be distributed by auto-download.
313323
namespace PakPath {

‎src/engine/qcommon/files.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -626,6 +626,10 @@ const char* FS_LoadedPaks()
626626
static char info[BIG_INFO_STRING];
627627
info[0] = '\0';
628628
for (const FS::LoadedPakInfo& x: FS::PakPath::GetLoadedPaks()) {
629+
bool isBuiltin = x.type == FS::pakType_t::PAK_ZIP && x.name[0] == '*';
630+
631+
if (isBuiltin)
632+
continue;
629633
if (!x.pathPrefix.empty())
630634
continue;
631635
if (info[0])

0 commit comments

Comments
 (0)