Skip to content
This repository has been archived by the owner on Jan 31, 2024. It is now read-only.

Commit

Permalink
Remove obsolete model parameters, more comments
Browse files Browse the repository at this point in the history
  • Loading branch information
saddam213 committed Sep 8, 2023
1 parent fbce228 commit 2fe84a5
Show file tree
Hide file tree
Showing 8 changed files with 347 additions and 29 deletions.
77 changes: 77 additions & 0 deletions LLamaStack.Core/Config/IInferenceConfig.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,98 @@ namespace LLamaStack.Core.Config
{
public interface IInferenceConfig
{
/// <summary>
/// Gets or sets the penalty applied to token frequency, affecting token selection during language model inference.
/// </summary>
float FrequencyPenalty { get; set; }

/// <summary>
/// Gets or sets a list of <see cref="LLamaStack.Core.Models.LogitBiasModel"/> objects that provide bias information for specific tokens in the language model's vocabulary
/// </summary>
List<LogitBiasModel> LogitBias { get; set; }


/// <summary>
/// Gets or sets the maximum number of tokens to generate during inference, limiting the length of the generated text
/// </summary>
int MaxTokens { get; set; }


/// <summary>
/// Gets or sets the type of sampling strategy to use during language model inference (e.g., greedy, top-k, top-p)
/// </summary>
SamplerType SamplerType { get; set; }


/// <summary>
/// Gets or sets the mirostat eta used to adjust the strength of the Mirostat bias
/// </summary>
float MirostatEta { get; set; }


/// <summary>
/// Gets or sets the temperature or sensitivity of the Mirostat sampling process
/// </summary>
float MirostatTau { get; set; }


/// <summary>
/// Determines whether to apply penalty for generating newline characters ("\n") in the generated text.
/// </summary>
bool PenalizeNL { get; set; }


/// <summary>
/// Gets or sets the penalty applied to token presence in the generated text
/// </summary>
float PresencePenalty { get; set; }


/// <summary>
/// Gets or sets the number of tokens to repeat at the end of the generated text.
/// </summary>
int RepeatLastTokensCount { get; set; }


/// <summary>
/// Gets or sets the penalty applied for repeating tokens in the generated text
/// </summary>
float RepeatPenalty { get; set; }


/// <summary>
/// Gets or sets the temperature parameter for temperature-based sampling. Higher values make output more random, while lower values make it more deterministic
/// </summary>
float Temperature { get; set; }


/// <summary>
/// Gets or sets the parameter (z) used in the TFS (Top Few Sampling) strategy.
/// </summary>
float TfsZ { get; set; }


/// <summary>
/// Gets or sets the number of tokens to keep from the input text when generating output.
/// </summary>
int TokensKeep { get; set; }


/// <summary>
/// Gets or sets the maximum number of tokens to consider during top-k sampling
/// </summary>
int TopK { get; set; }


/// <summary>
/// Gets or sets the cumulative probability threshold for top-p sampling
/// </summary>
float TopP { get; set; }


/// <summary>
/// Gets or sets the typicality penalty applied during language model inference.
/// </summary>
float TypicalP { get; set; }
}
}
38 changes: 38 additions & 0 deletions LLamaStack.Core/Config/ISessionConfig.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,54 @@

namespace LLamaStack.Core.Config
{
/// <summary>
/// Interface for Session configurations
/// </summary>
public interface ISessionConfig
{
/// <summary>
/// Gets or sets the model name to open the session on
/// </summary>
string Model { get; set; }

/// <summary>
/// Gets or sets the type of the executor to use for inference.
/// </summary>
ExecutorType ExecutorType { get; set; }

/// <summary>
/// Gets or sets the initial prompt to start the session with.
/// </summary>
string Prompt { get; set; }

/// <summary>
/// Gets or sets the input prefix for Instruct executors.
/// </summary>
string InputPrefix { get; set; }

/// <summary>
/// Gets or sets the input suffix for Instruct executors.
/// </summary>
string InputSuffix { get; set; }

/// <summary>
/// Gets or sets one or more anti-prompt words as CSV. (Combined with AntiPrompts)
/// </summary>
string AntiPrompt { get; set; }

/// <summary>
/// Gets or sets a list of anti-prompt words. (Combined with AntiPrompt)
/// </summary>
public List<string> AntiPrompts { get; set; }

/// <summary>
/// Gets or sets a list of words to remove from the output as CSV. (Combined with OutputFilters)
/// </summary>
string OutputFilter { get; set; }

/// <summary>
/// Gets or sets a list of words to remove from the output, (Combined with OutputFilter)
/// </summary>
public List<string> OutputFilters { get; set; }
}
}
68 changes: 68 additions & 0 deletions LLamaStack.Core/Config/InferenceConfig.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,23 +3,91 @@

namespace LLamaStack.Core.Config
{
/// <summary>
/// Concrete implemtation of IInferenceConfig
/// </summary>
/// <seealso cref="LLamaStack.Core.Config.IInferenceConfig" />
public class InferenceConfig : IInferenceConfig
{
/// <summary>
/// Gets or sets the number of tokens to keep from the input text when generating output.
/// </summary>
public int TokensKeep { get; set; } = 0;


/// <summary>
/// Gets or sets the maximum number of tokens to generate during inference, limiting the length of the generated text
/// </summary>
public int MaxTokens { get; set; } = -1;

/// <summary>
/// Gets or sets the maximum number of tokens to consider during top-k sampling
/// </summary>
public int TopK { get; set; } = 40;

/// <summary>
/// Gets or sets the cumulative probability threshold for top-p sampling
/// </summary>
public float TopP { get; set; } = 0.95f;

/// <summary>
/// Gets or sets the parameter (z) used in the TFS (Top Few Sampling) strategy.
/// </summary>
public float TfsZ { get; set; } = 1.0f;

/// <summary>
/// Gets or sets the typicality penalty applied during language model inference.
/// </summary>
public float TypicalP { get; set; } = 1.0f;

/// <summary>
/// Gets or sets the temperature parameter for temperature-based sampling. Higher values make output more random, while lower values make it more deterministic
/// </summary>
public float Temperature { get; set; } = 0.8f;

/// <summary>
/// Gets or sets the penalty applied for repeating tokens in the generated text
/// </summary>
public float RepeatPenalty { get; set; } = 1.1f;

/// <summary>
/// Gets or sets the number of tokens to repeat at the end of the generated text.
/// </summary>
public int RepeatLastTokensCount { get; set; } = 64;

/// <summary>
/// Gets or sets the penalty applied to token frequency, affecting token selection during language model inference.
/// </summary>
public float FrequencyPenalty { get; set; } = .0f;

/// <summary>
/// Gets or sets the penalty applied to token presence in the generated text
/// </summary>
public float PresencePenalty { get; set; } = .0f;

/// <summary>
/// Gets or sets the temperature or sensitivity of the Mirostat sampling process
/// </summary>
public float MirostatTau { get; set; } = 5.0f;

/// <summary>
/// Gets or sets the mirostat eta used to adjust the strength of the Mirostat bias
/// </summary>
public float MirostatEta { get; set; } = 0.1f;

/// <summary>
/// Determines whether to apply penalty for generating newline characters ("\n") in the generated text.
/// </summary>
public bool PenalizeNL { get; set; } = true;

/// <summary>
/// Gets or sets the type of sampling strategy to use during language model inference (e.g., greedy, top-k, top-p)
/// </summary>
public SamplerType SamplerType { get; set; } = SamplerType.Default;

/// <summary>
/// Gets or sets a list of <see cref="LLamaStack.Core.Models.LogitBiasModel" /> objects that provide bias information for specific tokens in the language model's vocabulary
/// </summary>
public List<LogitBiasModel> LogitBias { get; set; } = new List<LogitBiasModel>();
}
}
19 changes: 19 additions & 0 deletions LLamaStack.Core/Config/LLamaStackConfig.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,31 @@

namespace LLamaStack.Core.Config
{
/// <summary>
/// LLamaStack appsettings.json config element
/// </summary>
/// <seealso cref="LLamaStack.Core.Config.IConfigSection" />
public class LLamaStackConfig : IConfigSection
{
/// <summary>
/// Gets or sets the ModelLoad type
/// </summary>
public ModelLoadType ModelLoadType { get; set; }

/// <summary>
/// Gets or sets the model state path.
/// </summary>
public string ModelStatePath { get; set; }


/// <summary>
/// Gets or sets the models.
/// </summary>
public List<ModelConfig> Models { get; set; }

/// <summary>
/// Perform any initialization, called directly after deserialization
/// </summary>
public void Initialize()
{
if (string.IsNullOrEmpty(ModelStatePath))
Expand Down
Loading

0 comments on commit 2fe84a5

Please sign in to comment.