Skip to content

Commit 7b0725e

Browse files
illwieckzillwieckz
committed
FileSystem: implement internal paks
1 parent 945511f commit 7b0725e

File tree

3 files changed

+141
-21
lines changed

3 files changed

+141
-21
lines changed

‎src/common/FileSystem.cpp‎

Lines changed: 131 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,35 @@ static Cvar::Cvar<bool> fs_legacypaks("fs_legacypaks", "also load pk3s, ignoring
129129
static Cvar::Cvar<int> fs_maxSymlinkDepth("fs_maxSymlinkDepth", "max depth of symlinks in zip paks (0 means disabled)", Cvar::NONE, 1);
130130
static Cvar::Cvar<std::string> fs_pakprefixes("fs_pakprefixes", "prefixes to look for paks to load", 0, "");
131131

132+
builtinPakMap_t builtinPakMap = {};
133+
134+
std::vector<PakInfo> builtinPaks = {};
135+
136+
void ClearBuiltinPaks()
137+
{
138+
builtinPakMap.clear();
139+
140+
builtinPaks.clear();
141+
builtinPaks.shrink_to_fit();
142+
}
143+
144+
void AddBuiltinPak(const std::string& name, const std::string& version, const builtinFileMap_t& map)
145+
{
146+
PakInfo *pak = new PakInfo();
147+
pak->name = Str::Format( "*%s", name );
148+
pak->version = version;
149+
pak->checksum = 0;
150+
pak->type = pakType_t::PAK_ZIP;
151+
pak->path = Str::Format( "*%s_%s", name, version );
152+
153+
builtinPaks.push_back(*pak);
154+
155+
Log::Debug("Adding builtin pak with %d files: %s", map.size(), pak->name);
156+
builtinPakMap[pak->name] = map;
157+
158+
PakPath::LoadPak(*pak);
159+
}
160+
132161
bool UseLegacyPaks()
133162
{
134163
return fs_legacypaks.Get();
@@ -1257,8 +1286,13 @@ static void InternalLoadPak(
12571286
bool hasDeps = false;
12581287
offset_t depsOffset = 0;
12591288
ZipArchive zipFile;
1289+
12601290
bool isLegacy = pak.version.empty();
12611291

1292+
bool isBuiltin = pak.type == pakType_t::PAK_ZIP && pak.name[ 0 ] == '*';
1293+
bool isDir = pak.type == pakType_t::PAK_DIR;
1294+
bool isZip = pak.type == pakType_t::PAK_ZIP && !isBuiltin;
1295+
12621296
// Check if this pak has already been loaded to avoid recursive dependencies
12631297
for (auto& x: loadedPaks) {
12641298
// If the prefix is a superset of our current prefix, then it already
@@ -1267,13 +1301,15 @@ static void InternalLoadPak(
12671301
return;
12681302
}
12691303

1270-
if (pak.type == pakType_t::PAK_ZIP) {
1304+
if (isBuiltin) {
1305+
fsLogs.WithoutSuppression().Notice("Loading builtin pak '%s'...", pak.path.c_str());
1306+
} else if (isZip) {
12711307
if (!isLegacy) {
12721308
fsLogs.WithoutSuppression().Notice("Loading pak '%s'...", pak.path.c_str());
12731309
} else {
12741310
fsLogs.WithoutSuppression().Notice("Loading legacy pak '%s'...", pak.path.c_str());
12751311
}
1276-
} else if (pak.type == pakType_t::PAK_DIR) {
1312+
} else if (isDir) {
12771313
if (!isLegacy) {
12781314
fsLogs.WithoutSuppression().Notice("Loading pakdir '%s'...", pak.path.c_str());
12791315
} else {
@@ -1292,7 +1328,13 @@ static void InternalLoadPak(
12921328
loadedPak.path = pak.path;
12931329

12941330
// Update the list of files, but don't overwrite existing files, so the sort order is preserved
1295-
if (pak.type == pakType_t::PAK_DIR) {
1331+
if (isBuiltin) {
1332+
for (auto& it : builtinPakMap[pak.name])
1333+
{
1334+
Log::Debug("Adding file from builtin pak %s: %s", pak.name, it.first);
1335+
fileMap.emplace(it.first, std::pair<uint32_t, offset_t>(loadedPaks.size() - 1, 0));
1336+
}
1337+
} else if (isDir) {
12961338
loadedPak.fd = -1;
12971339
auto dirRange = RawPath::ListFilesRecursive(pak.path, err);
12981340
if (err)
@@ -1316,7 +1358,7 @@ static void InternalLoadPak(
13161358
if (err)
13171359
return;
13181360
}
1319-
} else if (pak.type == pakType_t::PAK_ZIP) {
1361+
} else if (isZip) {
13201362
// Open file
13211363
loadedPak.fd = my_open(pak.path, openMode_t::MODE_READ);
13221364
if (loadedPak.fd == -1) {
@@ -1380,16 +1422,16 @@ static void InternalLoadPak(
13801422
// Directories (aka a dpkdir) don't need timestamp.
13811423
// Fixes Windows bug where calling _wstat64i with trailing slash causes "file not found" error.
13821424
// For future stat calls on directories, trim the trailing slash (if exists)
1383-
if (pak.type == pakType_t::PAK_ZIP) {
1425+
if (isZip) {
13841426
loadedPak.timestamp = FS::RawPath::FileTimestamp(pak.path, err);
13851427
if (err)
13861428
return;
13871429
}
13881430

13891431
loadedPak.pathPrefix = pathPrefix;
13901432

1391-
// Legacy paks don't have version neither checksum
1392-
if (!isLegacy) {
1433+
// Legacy and builtin paks don't have version neither checksum
1434+
if (!isLegacy && !isBuiltin) {
13931435
// If an explicit checksum was requested, verify that the pak we loaded is the one we are expecting
13941436
if (expectedChecksum && realChecksum != *expectedChecksum) {
13951437
SetErrorCodeFilesystem(err, filesystem_error::wrong_pak_checksum, pak.path);
@@ -1406,14 +1448,16 @@ static void InternalLoadPak(
14061448
if (!isLegacy) {
14071449
if (hasDeleted) {
14081450
std::string deletedData;
1409-
if (pak.type == pakType_t::PAK_DIR) {
1451+
if (isBuiltin) {
1452+
// Not implemented.
1453+
} else if (isDir) {
14101454
File depsFile = RawPath::OpenRead(Path::Build(pak.path, PAK_DELETED_FILE), err);
14111455
if (err)
14121456
return;
14131457
deletedData = depsFile.ReadAll(err);
14141458
if (err)
14151459
return;
1416-
} else if (pak.type == pakType_t::PAK_ZIP) {
1460+
} else if (isZip) {
14171461
zipFile.OpenFile(deletedOffset, err);
14181462
if (err)
14191463
return;
@@ -1434,14 +1478,16 @@ static void InternalLoadPak(
14341478
// Load dependencies (non-legacy paks (pk3) only)
14351479
if (loadDeps && hasDeps) {
14361480
std::string depsData;
1437-
if (pak.type == pakType_t::PAK_DIR) {
1481+
if (isBuiltin) {
1482+
// Not implemented.
1483+
} else if (isDir) {
14381484
File depsFile = RawPath::OpenRead(Path::Build(pak.path, PAK_DEPS_FILE), err);
14391485
if (err)
14401486
return;
14411487
depsData = depsFile.ReadAll(err);
14421488
if (err)
14431489
return;
1444-
} else if (pak.type == pakType_t::PAK_ZIP) {
1490+
} else if (isZip) {
14451491
zipFile.OpenFile(depsOffset, err);
14461492
if (err)
14471493
return;
@@ -1491,6 +1537,7 @@ void ClearPaks()
14911537
close(x.fd);
14921538
}
14931539
loadedPaks.clear();
1540+
FS::ClearBuiltinPaks();
14941541
FS::RefreshPaks();
14951542
}
14961543
#else // BUILD_VM
@@ -1549,7 +1596,14 @@ std::string ReadFile(Str::StringRef path, std::error_code& err)
15491596
}
15501597

15511598
const LoadedPakInfo& pak = loadedPaks[it->second.first];
1552-
if (pak.type == pakType_t::PAK_DIR) {
1599+
1600+
bool isBuiltin = pak.type == pakType_t::PAK_ZIP && pak.name[0] == '*';
1601+
bool isDir = pak.type == pakType_t::PAK_DIR;
1602+
bool isZip = pak.type == pakType_t::PAK_ZIP && !isBuiltin;
1603+
1604+
if (isBuiltin) {
1605+
return builtinPakMap[pak.name][path];
1606+
} else if (isDir) {
15531607
// Open file
15541608
File file = RawPath::OpenRead(Path::Build(pak.path, it->first), err);
15551609
if (err)
@@ -1565,7 +1619,7 @@ std::string ReadFile(Str::StringRef path, std::error_code& err)
15651619
out.resize(length);
15661620
file.Read(&out[0], length, err);
15671621
return out;
1568-
} else if (pak.type == pakType_t::PAK_ZIP) {
1622+
} else if (isZip) {
15691623
// Open zip
15701624
ZipArchive zipFile = ZipArchive::Open(pak.fd, err);
15711625
if (err)
@@ -1604,12 +1658,19 @@ void CopyFile(Str::StringRef path, const File& dest, std::error_code& err)
16041658
}
16051659

16061660
const LoadedPakInfo& pak = loadedPaks[it->second.first];
1607-
if (pak.type == pakType_t::PAK_DIR) {
1661+
1662+
bool isBuiltin = pak.type == pakType_t::PAK_ZIP && pak.name[0] == '*';
1663+
bool isDir = pak.type == pakType_t::PAK_DIR;
1664+
bool isZip = pak.type == pakType_t::PAK_ZIP && !isBuiltin;
1665+
1666+
if (isBuiltin) {
1667+
// Not implemented
1668+
} else if (isDir) {
16081669
File file = RawPath::OpenRead(Path::Build(pak.path, it->first), err);
16091670
if (err)
16101671
return;
16111672
file.CopyTo(dest, err);
1612-
} else if (pak.type == pakType_t::PAK_ZIP) {
1673+
} else if (isZip) {
16131674
// Open zip
16141675
ZipArchive zipFile = ZipArchive::Open(pak.fd, err);
16151676
if (err)
@@ -1672,7 +1733,15 @@ std::chrono::system_clock::time_point FileTimestamp(Str::StringRef path, std::er
16721733
}
16731734

16741735
const LoadedPakInfo& pak = loadedPaks[it->second.first];
1675-
if (pak.type == pakType_t::PAK_DIR) {
1736+
1737+
bool isBuiltin = pak.type == pakType_t::PAK_ZIP && pak.name[0] == '*';
1738+
bool isDir = pak.type == pakType_t::PAK_DIR;
1739+
bool isZip = pak.type == pakType_t::PAK_ZIP && !isBuiltin;
1740+
1741+
if (isBuiltin) {
1742+
// Not implemented.
1743+
return {};
1744+
} else if (isDir) {
16761745
#ifdef BUILD_VM
16771746
Util::optional<uint64_t> result;
16781747
VM::SendMsg<VM::FSPakPathTimestampMsg>(it->second.first, it->first, result);
@@ -1686,7 +1755,7 @@ std::chrono::system_clock::time_point FileTimestamp(Str::StringRef path, std::er
16861755
#else
16871756
return RawPath::FileTimestamp(Path::Build(pak.path, it->first), err);
16881757
#endif
1689-
} else if (pak.type == pakType_t::PAK_ZIP) {
1758+
} else if (isZip) {
16901759
return pak.timestamp;
16911760
}
16921761

@@ -2636,6 +2705,10 @@ void RefreshPaks()
26362705
if (a.checksum != b.checksum)
26372706
return a.checksum > b.checksum;
26382707

2708+
// Prefer builtin packages to zip packages.
2709+
if (b.type == pakType_t::PAK_ZIP && a.type == pakType_t::PAK_ZIP && b.name[0] == '*')
2710+
return true;
2711+
26392712
// Prefer zip packages to directory packages
26402713
if (b.type == pakType_t::PAK_ZIP && a.type != pakType_t::PAK_ZIP)
26412714
return true;
@@ -2660,6 +2733,16 @@ static const PakInfo* FindPakNoPrefix(Str::StringRef name)
26602733

26612734
const PakInfo* FindPak(Str::StringRef name)
26622735
{
2736+
#if defined(BUILD_ENGINE)
2737+
for ( auto& builtinPak : builtinPaks)
2738+
{
2739+
if ( builtinPak.name == name )
2740+
{
2741+
return &builtinPak;
2742+
}
2743+
}
2744+
#endif
2745+
26632746
Cmd::Args pakprefixes(Cvar::GetValue("fs_pakprefixes"));
26642747
for (const std::string &pakprefix: pakprefixes)
26652748
{
@@ -2715,7 +2798,12 @@ const PakInfo* FindPak(Str::StringRef name, Str::StringRef version, uint32_t che
27152798
return checksum > pakInfo.checksum;
27162799
});
27172800

2718-
if (iter == availablePaks.begin() || (iter - 1)->name != name || (iter - 1)->version != version || !(iter - 1)->checksum || *(iter - 1)->checksum != checksum) {
2801+
if (iter == availablePaks.begin()
2802+
|| (iter - 1)->name != name
2803+
|| (iter - 1)->version != version
2804+
|| !(iter - 1)->checksum
2805+
|| *(iter - 1)->checksum != checksum) {
2806+
27192807
// Try again, but this time look for the pak without a checksum. We will verify the checksum later.
27202808
iter = std::upper_bound(availablePaks.begin(), availablePaks.end(), name, [version](Str::StringRef name1, const PakInfo& pakInfo) -> bool {
27212809
int result = name1.compare(pakInfo.name);
@@ -2728,7 +2816,23 @@ const PakInfo* FindPak(Str::StringRef name, Str::StringRef version, uint32_t che
27282816
});
27292817

27302818
// Only allow zip packages because directories don't have a checksum
2731-
if (iter == availablePaks.begin() || (iter - 1)->type == pakType_t::PAK_DIR || (iter - 1)->name != name || (iter - 1)->version != version || (iter - 1)->checksum)
2819+
if (iter == availablePaks.begin())
2820+
return nullptr;
2821+
2822+
PakInfo& pak = (iter - 1)[0];
2823+
2824+
bool isBuiltin = pak.type == pakType_t::PAK_ZIP && pak.name[ 0 ] == '*';
2825+
bool isDir = pak.type == pakType_t::PAK_DIR;
2826+
2827+
if (isBuiltin)
2828+
return nullptr;
2829+
if (isDir)
2830+
return nullptr;
2831+
if (pak.name != name)
2832+
return nullptr;
2833+
if (pak.version != version)
2834+
return nullptr;
2835+
if (pak.checksum)
27322836
return nullptr;
27332837
}
27342838

@@ -2944,12 +3048,19 @@ void HandleFileSystemSyscall(int minor, Util::Reader& reader, IPC::Channel& chan
29443048
case VM::FS_PAKPATH_TIMESTAMP:
29453049
IPC::HandleMsg<VM::FSPakPathTimestampMsg>(channel, std::move(reader), [](uint32_t pakIndex, std::string path, Util::optional<uint64_t>& out) {
29463050
auto& loadedPaks = FS::PakPath::GetLoadedPaks();
3051+
29473052
if (loadedPaks.size() <= pakIndex)
29483053
return;
2949-
if (loadedPaks[pakIndex].type == pakType_t::PAK_ZIP)
3054+
3055+
const LoadedPakInfo& pak = loadedPaks[pakIndex];
3056+
bool isDir = pak.type == pakType_t::PAK_DIR;
3057+
3058+
if (!isDir)
29503059
return;
3060+
29513061
if (!Path::IsValid(path, false))
29523062
return;
3063+
29533064
std::error_code err;
29543065
std::chrono::system_clock::time_point t = RawPath::FileTimestamp(Path::Build(loadedPaks[pakIndex].path, path), err);
29553066
if (!err)

‎src/common/FileSystem.h‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -265,7 +265,7 @@ template<typename State> class DirectoryIterator {
265265
// Type of pak
266266
enum class pakType_t {
267267
PAK_ZIP, // Zip archive
268-
PAK_DIR // Directory
268+
PAK_DIR, // Directory
269269
};
270270

271271
// Information about a package
@@ -308,6 +308,11 @@ struct LoadedPakInfo: public PakInfo {
308308
std::string pathPrefix;
309309
};
310310

311+
using builtinFileMap_t = std::unordered_map<std::string, std::string>;
312+
using builtinPakMap_t = std::unordered_map<std::string, builtinFileMap_t>;
313+
314+
void AddBuiltinPak(const std::string& name, const std::string& version, const builtinFileMap_t& map);
315+
311316
// Operations which work on files that are in packages. Packages should be used
312317
// for read-only assets which can be distributed by auto-download.
313318
namespace PakPath {

‎src/engine/qcommon/files.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -626,6 +626,10 @@ const char* FS_LoadedPaks()
626626
static char info[BIG_INFO_STRING];
627627
info[0] = '\0';
628628
for (const FS::LoadedPakInfo& x: FS::PakPath::GetLoadedPaks()) {
629+
bool isBuiltin = x.type == FS::pakType_t::PAK_ZIP && x.name[0] == '*';
630+
631+
if (isBuiltin)
632+
continue;
629633
if (!x.pathPrefix.empty())
630634
continue;
631635
if (info[0])

0 commit comments

Comments
 (0)