diff --git a/common/arg.cpp b/common/arg.cpp index a570810281499..c7a85a33c0129 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -1232,6 +1232,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, const std::string & value) { const auto sampler_names = string_split(value, ';'); params.sampling.samplers = common_sampler_types_from_names(sampler_names, true); + params.sampling.sampling_mask |= common_params_sampling::SAMPLING_MASK_BITS_SAMPLERS; } ).set_sparam()); add_opt(common_arg( @@ -1261,6 +1262,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, const std::string & value) { params.sampling.temp = std::stof(value); params.sampling.temp = std::max(params.sampling.temp, 0.0f); + params.sampling.sampling_mask |= common_params_sampling::SAMPLING_MASK_BITS_TEMP; } ).set_sparam()); add_opt(common_arg( @@ -1268,6 +1270,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex string_format("top-k sampling (default: %d, 0 = disabled)", params.sampling.top_k), [](common_params & params, int value) { params.sampling.top_k = value; + params.sampling.sampling_mask |= common_params_sampling::SAMPLING_MASK_BITS_TOP_K; } ).set_sparam()); add_opt(common_arg( @@ -1275,6 +1278,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex string_format("top-p sampling (default: %.1f, 1.0 = disabled)", (double)params.sampling.top_p), [](common_params & params, const std::string & value) { params.sampling.top_p = std::stof(value); + params.sampling.sampling_mask |= common_params_sampling::SAMPLING_MASK_BITS_TOP_P; } ).set_sparam()); add_opt(common_arg( @@ -1282,6 +1286,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex string_format("min-p sampling (default: %.1f, 0.0 = disabled)", (double)params.sampling.min_p), [](common_params & params, const std::string & value) { params.sampling.min_p = std::stof(value); + params.sampling.sampling_mask |= common_params_sampling::SAMPLING_MASK_BITS_MIN_P; } ).set_sparam()); add_opt(common_arg( @@ -1296,6 +1301,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex string_format("xtc probability (default: %.1f, 0.0 = disabled)", (double)params.sampling.xtc_probability), [](common_params & params, const std::string & value) { params.sampling.xtc_probability = std::stof(value); + params.sampling.sampling_mask |= common_params_sampling::SAMPLING_MASK_BITS_XTC_PROBABILITY; } ).set_sparam()); add_opt(common_arg( @@ -1303,6 +1309,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex string_format("xtc threshold (default: %.1f, 1.0 = disabled)", (double)params.sampling.xtc_threshold), [](common_params & params, const std::string & value) { params.sampling.xtc_threshold = std::stof(value); + params.sampling.sampling_mask |= common_params_sampling::SAMPLING_MASK_BITS_XTC_THRESHOLD; } ).set_sparam()); add_opt(common_arg( @@ -1321,6 +1328,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } params.sampling.penalty_last_n = value; params.sampling.n_prev = std::max(params.sampling.n_prev, params.sampling.penalty_last_n); + params.sampling.sampling_mask |= common_params_sampling::SAMPLING_MASK_BITS_PENALTY_LAST_N; } ).set_sparam()); add_opt(common_arg( @@ -1328,6 +1336,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex string_format("penalize repeat sequence of tokens (default: %.1f, 1.0 = disabled)", (double)params.sampling.penalty_repeat), [](common_params & params, const std::string & value) { params.sampling.penalty_repeat = std::stof(value); + params.sampling.sampling_mask |= common_params_sampling::SAMPLING_MASK_BITS_PENALTY_REPEAT; } ).set_sparam()); add_opt(common_arg( @@ -1425,6 +1434,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex "(default: %d, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0)", params.sampling.mirostat), [](common_params & params, int value) { params.sampling.mirostat = value; + params.sampling.sampling_mask |= common_params_sampling::SAMPLING_MASK_BITS_MIROSTAT; } ).set_sparam()); add_opt(common_arg( @@ -1432,6 +1442,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex string_format("Mirostat learning rate, parameter eta (default: %.1f)", (double)params.sampling.mirostat_eta), [](common_params & params, const std::string & value) { params.sampling.mirostat_eta = std::stof(value); + params.sampling.sampling_mask |= common_params_sampling::SAMPLING_MASK_BITS_MIROSTAT_ETA; } ).set_sparam()); add_opt(common_arg( @@ -1439,6 +1450,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex string_format("Mirostat target entropy, parameter tau (default: %.1f)", (double)params.sampling.mirostat_tau), [](common_params & params, const std::string & value) { params.sampling.mirostat_tau = std::stof(value); + params.sampling.sampling_mask |= common_params_sampling::SAMPLING_MASK_BITS_MIROSTAT_TAU; } ).set_sparam()); add_opt(common_arg( diff --git a/common/common.cpp b/common/common.cpp index a8d709ab1d050..0250024a4e497 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -8,6 +8,7 @@ #include "common.h" #include "log.h" #include "llama.h" +#include "sampling.h" #include #include @@ -946,6 +947,58 @@ std::vector fs_list_files(const std::string & path) { // Model utils // +static inline void common_init_sampler_from_model( + const llama_model * model, + common_params_sampling & sparams) { + + const uint16_t mask = sparams.sampling_mask; + + auto get_int32 = [&](const char * key, int32_t & dst, uint16_t user_override) { + if (mask & user_override) return; + + char buf[64] = {0}; + if (llama_model_meta_val_str(model, key, buf, sizeof(buf)) > 0) { + char * end = nullptr; + int32_t v = strtol(buf, &end, 10); + if (end && end != buf) dst = v; + } + }; + + auto get_float = [&](const char * key, float & dst, uint16_t user_override) { + if (mask & user_override) return; + + char buf[128] = {0}; + if (llama_model_meta_val_str(model, key, buf, sizeof(buf)) > 0) { + char * end = nullptr; + float v = strtof(buf, &end); + if (end && end != buf) dst = v; + } + }; + + // Sampler sequence + if (!(mask & common_params_sampling::SAMPLING_MASK_BITS_SAMPLERS)) { + char buf[512] = {0}; + if (llama_model_meta_val_str(model, "general.sampler.sequence", buf, sizeof(buf)) > 0) { + const std::vector sampler_names = string_split(std::string(buf), ';'); + if (!sampler_names.empty()) { + sparams.samplers = common_sampler_types_from_names(sampler_names, true); + } + } + } + + get_int32("general.sampler.top_k", sparams.top_k, common_params_sampling::SAMPLING_MASK_BITS_TOP_K); + get_float("general.sampler.top_p", sparams.top_p, common_params_sampling::SAMPLING_MASK_BITS_TOP_P); + get_float("general.sampler.min_p", sparams.min_p, common_params_sampling::SAMPLING_MASK_BITS_MIN_P); + get_float("general.sampler.xtc_probability", sparams.xtc_probability, common_params_sampling::SAMPLING_MASK_BITS_XTC_PROBABILITY); + get_float("general.sampler.xtc_threshold", sparams.xtc_threshold, common_params_sampling::SAMPLING_MASK_BITS_XTC_THRESHOLD); + get_float("general.sampler.temp", sparams.temp, common_params_sampling::SAMPLING_MASK_BITS_TEMP); + get_int32("general.sampler.penalty_last_n", sparams.penalty_last_n, common_params_sampling::SAMPLING_MASK_BITS_PENALTY_LAST_N); + get_float("general.sampler.penalty_repeat", sparams.penalty_repeat, common_params_sampling::SAMPLING_MASK_BITS_PENALTY_REPEAT); + get_int32("general.sampler.mirostat", sparams.mirostat, common_params_sampling::SAMPLING_MASK_BITS_MIROSTAT); + get_float("general.sampler.mirostat_tau", sparams.mirostat_tau, common_params_sampling::SAMPLING_MASK_BITS_MIROSTAT_TAU); + get_float("general.sampler.mirostat_eta", sparams.mirostat_eta, common_params_sampling::SAMPLING_MASK_BITS_MIROSTAT_ETA); +} + struct common_init_result common_init_from_params(common_params & params) { common_init_result iparams; auto mparams = common_model_params_to_llama(params); @@ -957,6 +1010,8 @@ struct common_init_result common_init_from_params(common_params & params) { return iparams; } + common_init_sampler_from_model(model, params.sampling); + const llama_vocab * vocab = llama_model_get_vocab(model); auto cparams = common_context_params_to_llama(params); diff --git a/common/common.h b/common/common.h index 8540725aaa476..19a708b2452c2 100644 --- a/common/common.h +++ b/common/common.h @@ -165,6 +165,22 @@ struct common_params_sampling { bool no_perf = false; // disable performance metrics bool timing_per_token = false; + uint16_t sampling_mask = 0; // bitfield to track user-specified samplers + enum sampling_mask_bits : uint16_t { + SAMPLING_MASK_BITS_SAMPLERS = 1 << 0, + SAMPLING_MASK_BITS_TOP_K = 1 << 1, + SAMPLING_MASK_BITS_TOP_P = 1 << 2, + SAMPLING_MASK_BITS_MIN_P = 1 << 3, + SAMPLING_MASK_BITS_XTC_PROBABILITY = 1 << 4, + SAMPLING_MASK_BITS_XTC_THRESHOLD = 1 << 5, + SAMPLING_MASK_BITS_TEMP = 1 << 6, + SAMPLING_MASK_BITS_PENALTY_LAST_N = 1 << 7, + SAMPLING_MASK_BITS_PENALTY_REPEAT = 1 << 8, + SAMPLING_MASK_BITS_MIROSTAT = 1 << 9, + SAMPLING_MASK_BITS_MIROSTAT_TAU = 1 << 10, + SAMPLING_MASK_BITS_MIROSTAT_ETA = 1 << 11, + }; + std::vector dry_sequence_breakers = {"\n", ":", "\"", "*"}; // default sequence breakers for DRY diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 6b4b6c5ab075d..be374c678cf30 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -25,6 +25,20 @@ class General: ALIGNMENT = "general.alignment" FILE_TYPE = "general.file_type" + # Recommended Sampler Parameters + SAMPLER_SEQUENCE = "general.sampler.sequence" + SAMPLER_TOP_K = "general.sampler.top_k" + SAMPLER_TOP_P = "general.sampler.top_p" + SAMPLER_MIN_P = "general.sampler.min_p" + SAMPLER_XTC_PROBABILITY = "general.sampler.xtc_probability" + SAMPLER_XTC_THRESHOLD = "general.sampler.xtc_threshold" + SAMPLER_TEMP = "general.sampler.temp" + SAMPLER_PENALTY_LAST_N = "general.sampler.penalty_last_n" + SAMPLER_PENALTY_REPEAT = "general.sampler.penalty_repeat" + SAMPLER_MIROSTAT = "general.sampler.mirostat" + SAMPLER_MIROSTAT_TAU = "general.sampler.mirostat_tau" + SAMPLER_MIROSTAT_ETA = "general.sampler.mirostat_eta" + # Authorship Metadata NAME = "general.name" AUTHOR = "general.author" diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index a051daeeb1341..067c062ba76c3 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -496,6 +496,42 @@ def add_custom_alignment(self, alignment: int) -> None: def add_file_type(self, ftype: int) -> None: self.add_uint32(Keys.General.FILE_TYPE, ftype) + def add_sampler_sequence(self, sequence: str) -> None: + self.add_string(Keys.General.SAMPLER_SEQUENCE, sequence) + + def add_sampler_top_k(self, top_k: int) -> None: + self.add_int32(Keys.General.SAMPLER_TOP_K, top_k) + + def add_sampler_top_p(self, top_p: float) -> None: + self.add_float32(Keys.General.SAMPLER_TOP_P, top_p) + + def add_sampler_min_p(self, min_p: float) -> None: + self.add_float32(Keys.General.SAMPLER_MIN_P, min_p) + + def add_sampler_xtc_probability(self, xtc_probability: float) -> None: + self.add_float32(Keys.General.SAMPLER_XTC_PROBABILITY, xtc_probability) + + def add_sampler_xtc_threshold(self, xtc_threshold: float) -> None: + self.add_float32(Keys.General.SAMPLER_XTC_THRESHOLD, xtc_threshold) + + def add_sampler_temp(self, temp: float) -> None: + self.add_float32(Keys.General.SAMPLER_TEMP, temp) + + def add_sampler_penalty_last_n(self, penalty_last_n: int) -> None: + self.add_int32(Keys.General.SAMPLER_PENALTY_LAST_N, penalty_last_n) + + def add_sampler_penalty_repeat(self, penalty_repeat: float) -> None: + self.add_float32(Keys.General.SAMPLER_PENALTY_REPEAT, penalty_repeat) + + def add_sampler_mirostat(self, mirostat: int) -> None: + self.add_int32(Keys.General.SAMPLER_MIROSTAT, mirostat) + + def add_sampler_mirostat_tau(self, mirostat_tau: float) -> None: + self.add_float32(Keys.General.SAMPLER_MIROSTAT_TAU, mirostat_tau) + + def add_sampler_mirostat_eta(self, mirostat_eta: float) -> None: + self.add_float32(Keys.General.SAMPLER_MIROSTAT_ETA, mirostat_eta) + def add_name(self, name: str) -> None: self.add_string(Keys.General.NAME, name) diff --git a/gguf-py/gguf/metadata.py b/gguf-py/gguf/metadata.py index 67efedbdbc564..2244c072fde3b 100644 --- a/gguf-py/gguf/metadata.py +++ b/gguf-py/gguf/metadata.py @@ -17,6 +17,20 @@ @dataclass class Metadata: + # Recommended Sampler Parameters to be written to GGUF KV Store + sampler_sequence: Optional[str] = None + sampler_top_k: Optional[int] = None + sampler_top_p: Optional[float] = None + sampler_min_p: Optional[float] = None + sampler_xtc_probability: Optional[float] = None + sampler_xtc_threshold: Optional[float] = None + sampler_temp: Optional[float] = None + sampler_penalty_last_n: Optional[int] = None + sampler_penalty_repeat: Optional[float] = None + sampler_mirostat: Optional[int] = None + sampler_mirostat_tau: Optional[float] = None + sampler_mirostat_eta: Optional[float] = None + # Authorship Metadata to be written to GGUF KV Store name: Optional[str] = None author: Optional[str] = None @@ -54,15 +68,43 @@ def load(metadata_override_path: Optional[Path] = None, model_path: Optional[Pat model_card = Metadata.load_model_card(model_path) hf_params = Metadata.load_hf_parameters(model_path) + gen_config = Metadata.load_generation_config(model_path) # TODO: load adapter_config.json when possible, it usually contains the base model of the LoRA adapter # heuristics metadata = Metadata.apply_metadata_heuristic(metadata, model_card, hf_params, model_path, total_params) + if gen_config: + metadata.sampler_sequence = gen_config.get("sequence", metadata.sampler_sequence) + metadata.sampler_top_k = gen_config.get("top_k", metadata.sampler_top_k) + metadata.sampler_top_p = gen_config.get("top_p", metadata.sampler_top_p) + metadata.sampler_min_p = gen_config.get("min_p", metadata.sampler_min_p) + metadata.sampler_xtc_probability = gen_config.get("xtc_probability", metadata.sampler_xtc_probability) + metadata.sampler_xtc_threshold = gen_config.get("xtc_threshold", metadata.sampler_xtc_threshold) + metadata.sampler_temp = gen_config.get("temperature", metadata.sampler_temp) + metadata.sampler_penalty_last_n = gen_config.get("penalty_last_n", metadata.sampler_penalty_last_n) + metadata.sampler_penalty_repeat = gen_config.get("penalty_repeat", metadata.sampler_penalty_repeat) + metadata.sampler_mirostat = gen_config.get("mirostat", metadata.sampler_mirostat) + metadata.sampler_mirostat_tau = gen_config.get("mirostat_tau", metadata.sampler_mirostat_tau) + metadata.sampler_mirostat_eta = gen_config.get("mirostat_eta", metadata.sampler_mirostat_eta) + # Metadata Override File Provided # This is based on LLM_KV_NAMES mapping in llama.cpp metadata_override = Metadata.load_metadata_override(metadata_override_path) + metadata.sampler_sequence = metadata_override.get(Keys.General.SAMPLER_SEQUENCE, metadata.sampler_sequence) + metadata.sampler_top_k = metadata_override.get(Keys.General.SAMPLER_TOP_K, metadata.sampler_top_k) + metadata.sampler_top_p = metadata_override.get(Keys.General.SAMPLER_TOP_P, metadata.sampler_top_p) + metadata.sampler_min_p = metadata_override.get(Keys.General.SAMPLER_MIN_P, metadata.sampler_min_p) + metadata.sampler_xtc_probability = metadata_override.get(Keys.General.SAMPLER_XTC_PROBABILITY, metadata.sampler_xtc_probability) + metadata.sampler_xtc_threshold = metadata_override.get(Keys.General.SAMPLER_XTC_THRESHOLD, metadata.sampler_xtc_threshold) + metadata.sampler_temp = metadata_override.get(Keys.General.SAMPLER_TEMP, metadata.sampler_temp) + metadata.sampler_penalty_last_n = metadata_override.get(Keys.General.SAMPLER_PENALTY_LAST_N, metadata.sampler_penalty_last_n) + metadata.sampler_penalty_repeat = metadata_override.get(Keys.General.SAMPLER_PENALTY_REPEAT, metadata.sampler_penalty_repeat) + metadata.sampler_mirostat = metadata_override.get(Keys.General.SAMPLER_MIROSTAT, metadata.sampler_mirostat) + metadata.sampler_mirostat_tau = metadata_override.get(Keys.General.SAMPLER_MIROSTAT_TAU, metadata.sampler_mirostat_tau) + metadata.sampler_mirostat_eta = metadata_override.get(Keys.General.SAMPLER_MIROSTAT_ETA, metadata.sampler_mirostat_eta) + metadata.name = metadata_override.get(Keys.General.NAME, metadata.name) metadata.author = metadata_override.get(Keys.General.AUTHOR, metadata.author) metadata.version = metadata_override.get(Keys.General.VERSION, metadata.version) @@ -172,6 +214,23 @@ def load_hf_parameters(model_path: Optional[Path] = None) -> dict[str, Any]: with open(config_path, "r", encoding="utf-8") as f: return json.load(f) + @staticmethod + def load_generation_config(model_path: Optional[Path] = None) -> dict[str, Any]: + if model_path is None or not model_path.is_dir(): + return {} + + generation_config_path = model_path / "generation_config.json" + + if not generation_config_path.is_file(): + return {} + + try: + with open(generation_config_path, "r", encoding="utf-8") as f: + return json.load(f) + except (json.JSONDecodeError, IOError): + # not all models have valid generation_config.json + return {} + @staticmethod def id_to_title(string): # Convert capitalization into title form unless acronym or version number @@ -546,6 +605,32 @@ def use_array_model_card_metadata(metadata_key: str, model_card_key: str): def set_gguf_meta_model(self, gguf_writer: gguf.GGUFWriter): assert self.name is not None + + if self.sampler_sequence is not None: + gguf_writer.add_sampler_sequence(self.sampler_sequence) + if self.sampler_top_k is not None: + gguf_writer.add_sampler_top_k(self.sampler_top_k) + if self.sampler_top_p is not None: + gguf_writer.add_sampler_top_p(self.sampler_top_p) + if self.sampler_min_p is not None: + gguf_writer.add_sampler_min_p(self.sampler_min_p) + if self.sampler_xtc_probability is not None: + gguf_writer.add_sampler_xtc_probability(self.sampler_xtc_probability) + if self.sampler_xtc_threshold is not None: + gguf_writer.add_sampler_xtc_threshold(self.sampler_xtc_threshold) + if self.sampler_temp is not None: + gguf_writer.add_sampler_temp(self.sampler_temp) + if self.sampler_penalty_last_n is not None: + gguf_writer.add_sampler_penalty_last_n(self.sampler_penalty_last_n) + if self.sampler_penalty_repeat is not None: + gguf_writer.add_sampler_penalty_repeat(self.sampler_penalty_repeat) + if self.sampler_mirostat is not None: + gguf_writer.add_sampler_mirostat(self.sampler_mirostat) + if self.sampler_mirostat_tau is not None: + gguf_writer.add_sampler_mirostat_tau(self.sampler_mirostat_tau) + if self.sampler_mirostat_eta is not None: + gguf_writer.add_sampler_mirostat_eta(self.sampler_mirostat_eta) + gguf_writer.add_name(self.name) if self.author is not None: diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index b7642b568dffb..423ddf7d41bde 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -112,19 +112,31 @@ static const std::map LLM_ARCH_NAMES = { }; static const std::map LLM_KV_NAMES = { - { LLM_KV_GENERAL_TYPE, "general.type" }, - { LLM_KV_GENERAL_ARCHITECTURE, "general.architecture" }, - { LLM_KV_GENERAL_QUANTIZATION_VERSION, "general.quantization_version" }, - { LLM_KV_GENERAL_ALIGNMENT, "general.alignment" }, - { LLM_KV_GENERAL_FILE_TYPE, "general.file_type" }, - { LLM_KV_GENERAL_NAME, "general.name" }, - { LLM_KV_GENERAL_AUTHOR, "general.author" }, - { LLM_KV_GENERAL_VERSION, "general.version" }, - { LLM_KV_GENERAL_URL, "general.url" }, - { LLM_KV_GENERAL_DESCRIPTION, "general.description" }, - { LLM_KV_GENERAL_LICENSE, "general.license" }, - { LLM_KV_GENERAL_SOURCE_URL, "general.source.url" }, - { LLM_KV_GENERAL_SOURCE_HF_REPO, "general.source.huggingface.repository" }, + { LLM_KV_GENERAL_TYPE, "general.type" }, + { LLM_KV_GENERAL_ARCHITECTURE, "general.architecture" }, + { LLM_KV_GENERAL_QUANTIZATION_VERSION, "general.quantization_version" }, + { LLM_KV_GENERAL_ALIGNMENT, "general.alignment" }, + { LLM_KV_GENERAL_FILE_TYPE, "general.file_type" }, + { LLM_KV_GENERAL_SAMPLER_SEQUENCE, "general.sampler.sequence" }, + { LLM_KV_GENERAL_SAMPLER_TOP_K, "general.sampler.top_k" }, + { LLM_KV_GENERAL_SAMPLER_TOP_P, "general.sampler.top_p" }, + { LLM_KV_GENERAL_SAMPLER_MIN_P, "general.sampler.min_p" }, + { LLM_KV_GENERAL_SAMPLER_XTC_PROBABILITY, "general.sampler.xtc_probability" }, + { LLM_KV_GENERAL_SAMPLER_XTC_THRESHOLD, "general.sampler.xtc_threshold" }, + { LLM_KV_GENERAL_SAMPLER_TEMP, "general.sampler.temperature" }, + { LLM_KV_GENERAL_SAMPLER_PENALTY_LAST_N, "general.sampler.penalty_last_n" }, + { LLM_KV_GENERAL_SAMPLER_PENALTY_REPEAT, "general.sampler.penalty_repeat" }, + { LLM_KV_GENERAL_SAMPLER_MIROSTAT, "general.sampler.mirostat" }, + { LLM_KV_GENERAL_SAMPLER_MIROSTAT_TAU, "general.sampler.mirostat_tau" }, + { LLM_KV_GENERAL_SAMPLER_MIROSTAT_ETA, "general.sampler.mirostat_eta" }, + { LLM_KV_GENERAL_NAME, "general.name" }, + { LLM_KV_GENERAL_AUTHOR, "general.author" }, + { LLM_KV_GENERAL_VERSION, "general.version" }, + { LLM_KV_GENERAL_URL, "general.url" }, + { LLM_KV_GENERAL_DESCRIPTION, "general.description" }, + { LLM_KV_GENERAL_LICENSE, "general.license" }, + { LLM_KV_GENERAL_SOURCE_URL, "general.source.url" }, + { LLM_KV_GENERAL_SOURCE_HF_REPO, "general.source.huggingface.repository" }, { LLM_KV_VOCAB_SIZE, "%s.vocab_size" }, { LLM_KV_CONTEXT_LENGTH, "%s.context_length" }, diff --git a/src/llama-arch.h b/src/llama-arch.h index a769dd1e85741..2f868263a9f17 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -121,6 +121,18 @@ enum llm_kv { LLM_KV_GENERAL_QUANTIZATION_VERSION, LLM_KV_GENERAL_ALIGNMENT, LLM_KV_GENERAL_FILE_TYPE, + LLM_KV_GENERAL_SAMPLER_SEQUENCE, + LLM_KV_GENERAL_SAMPLER_TOP_K, + LLM_KV_GENERAL_SAMPLER_TOP_P, + LLM_KV_GENERAL_SAMPLER_MIN_P, + LLM_KV_GENERAL_SAMPLER_XTC_PROBABILITY, + LLM_KV_GENERAL_SAMPLER_XTC_THRESHOLD, + LLM_KV_GENERAL_SAMPLER_TEMP, + LLM_KV_GENERAL_SAMPLER_PENALTY_LAST_N, + LLM_KV_GENERAL_SAMPLER_PENALTY_REPEAT, + LLM_KV_GENERAL_SAMPLER_MIROSTAT, + LLM_KV_GENERAL_SAMPLER_MIROSTAT_TAU, + LLM_KV_GENERAL_SAMPLER_MIROSTAT_ETA, LLM_KV_GENERAL_NAME, LLM_KV_GENERAL_AUTHOR, LLM_KV_GENERAL_VERSION,