Искусственный интеллект на C#: интеграция с OpenAI и локальные модели

public class EmbeddingService
{
    private readonly OpenAIClient _client;
    private readonly Dictionary<string, float[]> _embeddingsCache;
    private readonly ISemanticCache _semanticCache;
    
    public EmbeddingService(string apiKey)
    {
        _client = new OpenAIClient(apiKey);
        _embeddingsCache = new Dictionary<string, float[]>();
        _semanticCache = new MemorySemanticCache();
    }
    
    // Генерация эмбеддингов для текста
    public async Task<float[]> GetEmbeddingAsync(string text, string model = "text-embedding-3-small")
    {
        // Проверка кэша
        if (_embeddingsCache.TryGetValue(text, out var cachedEmbedding))
        {
            return cachedEmbedding;
        }
        
        var request = new EmbeddingRequest
        {
            Input = text,
            Model = model,
            Dimensions = 1536 // Можно уменьшить для экономии
        };
        
        var response = await _client.EmbeddingsEndpoint.CreateEmbeddingAsync(request);
        var embedding = response.First().Embedding.ToArray();
        
        // Кэширование
        _embeddingsCache[text] = embedding;
        
        return embedding;
    }
    
    // Косинусное сходство
    public float CosineSimilarity(float[] vector1, float[] vector2)
    {
        if (vector1.Length != vector2.Length)
            throw new ArgumentException("Vectors must have the same length");
        
        float dotProduct = 0;
        float magnitude1 = 0;
        float magnitude2 = 0;
        
        for (int i = 0; i < vector1.Length; i++)
        {
            dotProduct += vector1[i] * vector2[i];
            magnitude1 += vector1[i] * vector1[i];
            magnitude2 += vector2[i] * vector2[i];
        }
        
        magnitude1 = (float)Math.Sqrt(magnitude1);
        magnitude2 = (float)Math.Sqrt(magnitude2);
        
        if (magnitude1 == 0 || magnitude2 == 0)
            return 0;
        
        return dotProduct / (magnitude1 * magnitude2);
    }
    
    // Семантический поиск
    public async Task<List<SearchResult>> SemanticSearchAsync(
        string query, 
        List<Document> documents, 
        int topK = 5)
    {
        var queryEmbedding = await GetEmbeddingAsync(query);
        var results = new List<SearchResult>();
        
        foreach (var doc in documents)
        {
            var docEmbedding = await GetEmbeddingAsync(doc.Content);
            var similarity = CosineSimilarity(queryEmbedding, docEmbedding);
            
            results.Add(new SearchResult
            {
                Document = doc,
                Similarity = similarity
            });
        }
        
        return results
            .OrderByDescending(r => r.Similarity)
            .Take(topK)
            .ToList();
    }
    
    // Кластеризация документов
    public async Task<List<DocumentCluster>> ClusterDocumentsAsync(
        List<Document> documents, 
        int numberOfClusters = 5)
    {
        // Генерация эмбеддингов для всех документов
        var embeddings = new List<float[]>();
        foreach (var doc in documents)
        {
            var embedding = await GetEmbeddingAsync(doc.Content);
            embeddings.Add(embedding);
        }
        
        // K-means кластеризация
        var clusters = await KMeansClusteringAsync(embeddings, numberOfClusters);
        
        // Группировка документов по кластерам
        var documentClusters = new List<DocumentCluster>();
        
        for (int i = 0; i < clusters.Count; i++)
        {
            var clusterDocs = new List<Document>();
            for (int j = 0; j < clusters[i].Count; j++)
            {
                clusterDocs.Add(documents[clusters[i][j]]);
            }
            
            documentClusters.Add(new DocumentCluster
            {
                Id = i,
                Documents = clusterDocs,
                Center = CalculateCentroid(embeddings.Where((_, idx) => clusters[i].Contains(idx)))
            });
        }
        
        return documentClusters;
    }
    
    private async Task<List<List<int>>> KMeansClusteringAsync(
        List<float[]> embeddings, 
        int k, 
        int maxIterations = 100)
    {
        // Инициализация центроидов случайными точками
        var random = new Random();
        var centroids = new List<float[]>();
        
        for (int i = 0; i < k; i++)
        {
            centroids.Add(embeddings[random.Next(embeddings.Count)]);
        }
        
        var clusters = new List<List<int>>();
        
        for (int iteration = 0; iteration < maxIterations; iteration++)
        {
            // Сброс кластеров
            clusters = Enumerable.Range(0, k)
                .Select(_ => new List<int>())
                .ToList();
            
            // Назначение точек ближайшим центроидам
            for (int i = 0; i < embeddings.Count; i++)
            {
                var distances = centroids
                    .Select((c, idx) => new 
                    { 
                        Index = idx, 
                        Distance = EuclideanDistance(embeddings[i], c) 
                    })
                    .OrderBy(d => d.Distance)
                    .First();
                
                clusters[distances.Index].Add(i);
            }
            
            // Пересчет центроидов
            var newCentroids = new List<float[]>();
            
            for (int i = 0; i < k; i++)
            {
                if (clusters[i].Count > 0)
                {
                    var clusterEmbeddings = clusters[i]
                        .Select(idx => embeddings[idx])
                        .ToList();
                    
                    newCentroids.Add(CalculateCentroid(clusterEmbeddings));
                }
                else
                {
                    // Если кластер пуст, инициализируем случайно
                    newCentroids.Add(embeddings[random.Next(embeddings.Count)]);
                }
            }
            
            // Проверка сходимости
            if (CentroidsConverged(centroids, newCentroids, threshold: 0.001f))
            {
                break;
            }
            
            centroids = newCentroids;
        }
        
        return clusters;
    }
    
    private float[] CalculateCentroid(IEnumerable<float[]> vectors)
    {
        var first = vectors.First();
        var centroid = new float[first.Length];
        
        foreach (var vector in vectors)
        {
            for (int i = 0; i < vector.Length; i++)
            {
                centroid[i] += vector[i];
            }
        }
        
        int count = vectors.Count();
        for (int i = 0; i < centroid.Length; i++)
        {
            centroid[i] /= count;
        }
        
        return centroid;
    }
    
    private float EuclideanDistance(float[] a, float[] b)
    {
        float sum = 0;
        for (int i = 0; i < a.Length; i++)
        {
            float diff = a[i] - b[i];
            sum += diff * diff;
        }
        return (float)Math.Sqrt(sum);
    }
    
    private bool CentroidsConverged(
        List<float[]> oldCentroids, 
        List<float[]> newCentroids, 
        float threshold)
    {
        for (int i = 0; i < oldCentroids.Count; i++)
        {
            if (EuclideanDistance(oldCentroids[i], newCentroids[i]) > threshold)
            {
                return false;
            }
        }
        return true;
    }
}

Локальные модели ИИ на C#

2.1. Использование ONNX Runtime

2.1.1. Загрузка и выполнение ONNX моделей

// Установка: dotnet add package Microsoft.ML.OnnxRuntime
//          dotnet add package Microsoft.ML.OnnxRuntime.Managed

using Microsoft.ML.OnnxRuntime;
using Microsoft.ML.OnnxRuntime.Tensors;

public class OnnxModelService : IDisposable
{
    private readonly InferenceSession _session;
    private readonly ILogger<OnnxModelService> _logger;
    private readonly Tokenizer _tokenizer;
    
    public OnnxModelService(string modelPath, ILogger<OnnxModelService> logger)
    {
        var options = new SessionOptions
        {
            LogSeverityLevel = OrtLoggingLevel.ORT_LOGGING_LEVEL_WARNING,
            GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL,
            ExecutionMode = ExecutionMode.ORT_PARALLEL,
            InterOpNumThreads = Environment.ProcessorCount,
            IntraOpNumThreads = Environment.ProcessorCount
        };
        
        // Использование CUDA если доступно
        if (HasCuda())
        {
            options.AppendExecutionProvider_CUDA();
        }
        else
        {
            options.AppendExecutionProvider_CPU();
        }
        
        _session = new InferenceSession(modelPath, options);
        _logger = logger;
        _tokenizer = new Tokenizer();
        
        LogModelInfo();
    }
    
    private bool HasCuda()
    {
        try
        {
            var cudaSessionOptions = SessionOptions.MakeSessionOptionWithCudaProvider();
            return true;
        }
        catch
        {
            return false;
        }
    }
    
    private void LogModelInfo()
    {
        _logger.LogInformation($"Model inputs: {_session.InputMetadata.Count}");
        _logger.LogInformation($"Model outputs: {_session.OutputMetadata.Count}");
        
        foreach (var input in _session.InputMetadata)
        {
            _logger.LogInformation($"Input: {input.Key}, Type: {input.Value.ElementType}, " +
                                 $"Shape: {string.Join(",", input.Value.Dimensions)}");
        }
    }
    
    // Текстовая генерация с локальной моделью
    public async Task<string> GenerateTextAsync(string prompt, int maxLength = 100)
    {
        var tokens = _tokenizer.Encode(prompt);
        
        var inputTensor = new DenseTensor<long>(
            new[] { 1, tokens.Length }, 
            new[] { tokens.Select(t => (long)t).ToArray() });
        
        var inputs = new List<NamedOnnxValue>
        {
            NamedOnnxValue.CreateFromTensor("input_ids", inputTensor)
        };
        
        var generatedText = prompt;
        
        for (int i = 0; i < maxLength; i++)
        {
            using var results = _session.Run(inputs);
            
            var logits = results.First().AsTensor<float>();
            var nextToken = SampleToken(logits);
            
            if (nextToken == _tokenizer.EosTokenId)
                break;
            
            generatedText += _tokenizer.Decode(new[] { nextToken });
            
            // Обновление inputs для следующей итерации
            tokens = tokens.Append(nextToken).ToArray();
            inputTensor = new DenseTensor<long>(
                new[] { 1, tokens.Length }, 
                new[] { tokens.Select(t => (long)t).ToArray() });
            
            inputs[0] = NamedOnnxValue.CreateFromTensor("input_ids", inputTensor);
        }
        
        return generatedText;
    }
    
    private int SampleToken(Tensor<float> logits)
    {
        // Реализация sampling с temperature
        var temperature = 0.7f;
        var probabilities = new float[logits.Length];
        
        for (int i = 0; i < logits.Length; i++)
        {
            probabilities[i] = (float)Math.Exp(logits[i] / temperature);
        }
        
        // Softmax
        var sum = probabilities.Sum();
        for (int i = 0; i < probabilities.Length; i++)
        {
            probabilities[i] /= sum;
        }
        
        // Выбор токена на основе вероятностей
        var random = new Random();
        var randomValue = random.NextDouble();
        var cumulative = 0.0;
        
        for (int i = 0; i < probabilities.Length; i++)
        {
            cumulative += probabilities[i];
            if (randomValue <= cumulative)
            {
                return i;
            }
        }
        
        return probabilities.Length - 1;
    }
    
    // Классификация текста
    public async Task<ClassificationResult> ClassifyTextAsync(string text)
    {
        var tokens = _tokenizer.Encode(text);
        
        var inputIds = new DenseTensor<long>(
            new[] { 1, tokens.Length }, 
            new[] { tokens.Select(t => (long)t).ToArray() });
        
        var attentionMask = new DenseTensor<long>(
            new[] { 1, tokens.Length },
            Enumerable.Repeat(1L, tokens.Length).ToArray());
        
        var inputs = new List<NamedOnnxValue>
        {
            NamedOnnxValue.CreateFromTensor("input_ids", inputIds),
            NamedOnnxValue.CreateFromTensor("attention_mask", attentionMask)
        };
        
        using var results = _session.Run(inputs);
        var logits = results.First().AsTensor<float>();
        
        var probabilities = Softmax(logits.ToArray());
        var predictedClass = Array.IndexOf(probabilities, probabilities.Max());
        
        return new ClassificationResult
        {
            ClassId = predictedClass,
            Confidence = probabilities[predictedClass],
            Probabilities = probabilities
        };
    }
    
    private float[] Softmax(float[] logits)
    {
        var max = logits.Max();
        var exp = logits.Select(x => (float)Math.Exp(x - max)).ToArray();
        var sum = exp.Sum();
        
        return exp.Select(x => x / sum).ToArray();
    }
    
    // Пакетная обработка
    public async Task<List<string>> ProcessBatchAsync(List<string> texts)
    {
        var batchSize = 8;
        var results = new List<string>();
        
        for (int i = 0; i < texts.Count; i += batchSize)
        {
            var batch = texts.Skip(i).Take(batchSize).ToList();
            var batchResults = await ProcessSingleBatchAsync(batch);
            results.AddRange(batchResults);
            
            _logger.LogInformation($"Processed batch {i / batchSize + 1}/" +
                                 $"{Math.Ceiling((double)texts.Count / batchSize)}");
        }
        
        return results;
    }
    
    private async Task<List<string>> ProcessSingleBatchAsync(List<string> batch)
    {
        // Подготовка batch inputs
        var maxLength = batch.Max(t => _tokenizer.Encode(t).Length);
        var batchSize = batch.Count;
        
        var inputIds = new DenseTensor<long>(new[] { batchSize, maxLength });
        var attentionMask = new DenseTensor<long>(new[] { batchSize, maxLength });
        
        for (int i = 0; i < batch.Count; i++)
        {
            var tokens = _tokenizer.Encode(batch[i]);
            
            for (int j = 0; j < tokens.Length; j++)
            {
                inputIds[i, j] = tokens[j];
                attentionMask[i, j] = 1;
            }
            
            // Padding
            for (int j = tokens.Length; j < maxLength; j++)
            {
                inputIds[i, j] = _tokenizer.PadTokenId;
                attentionMask[i, j] = 0;
            }
        }
        
        var inputs = new List<NamedOnnxValue>
        {
            NamedOnnxValue.CreateFromTensor("input_ids", inputIds),
            NamedOnnxValue.CreateFromTensor("attention_mask", attentionMask)
        };
        
        using var results = _session.Run(inputs);
        var logits = results.First().AsTensor<float>();
        
        // Обработка результатов batch
        var batchResults = new List<string>();
        
        for (int i = 0; i < batchSize; i++)
        {
            var slice = logits.GetSubTensor(i);
            var generated = ProcessSingleResult(slice);
            batchResults.Add(generated);
        }
        
        return batchResults;
    }
    
    public void Dispose()
    {
        _session?.Dispose();
    }
}

2.2. Локальные эмбеддинг-модели

2.2.1. Использование Sentence Transformers в C#