#5406 AI Agents revamp: vector store

inc/classes/BxDolAI.php

@@ -70,6 +70,104 @@ class BxDolAI extends BxDolFactory implements iBxDolSingleton
         return $GLOBALS['bxDolClasses'][__CLASS__];
     }
 
+    public static function getAiProviderInstance(int $iId):NeuronAI\Providers\AIProviderInterface
+    {
+        return self::getInstance()->getAiProviderInstance($iId);
+    }   
+
+    public static function getAiEmbeddingsProviderInstance(int $iId):NeuronAI\RAG\Embeddings\EmbeddingsProviderInterface
+    {
+        return self::getInstance()->getAiProviderInstance($iId);
+    }   
+
+    public static function getModelInstance(int $iId): NeuronAI\Providers\AIProviderInterface | NeuronAI\RAG\Embeddings\EmbeddingsProviderInterface
+    {
+        if (isset($GLOBALS['bxDolClasses'][__CLASS__ . '_Model_' . $iId]))
+            return $GLOBALS['bxDolClasses'][__CLASS__ . '_Model_' . $iId];
+
+        $aProvidersWithKey = ['anthropic'];
+        $a = BxDolAIQuery::getModelObject($iId);
+        if (!$a) {
+            bx_log('sys_agents', "Agent AI Model with id {$iId} not found", BX_LOG_ERR);
+            throw new Exception("Agent AI Model with id {$iId} not found");
+        }
+        if (in_array($a['type'], $aProvidersWithKey) && empty($a['key'])) {
+            bx_log('sys_agents', "Model with id {$iId} has empty key, can't be used", BX_LOG_ERR);
+            throw new Exception("Model with id {$iId} has empty key, can't be used");
+        }
+
+        if (!$a['active']) {
+            bx_log('sys_agents', "Model with id {$iId} is not active, can't be used", BX_LOG_ERR);
+            throw new Exception("Model with id {$iId} is not active, can't be used");
+        }
+
+        $aParameters = !empty($a['params']) ? json_decode($a['params'], true) : [];
+
+        switch($a['type']) {
+            // TODO: add more providers
+            case 'anthropic':
+                $o = new NeuronAI\Providers\Anthropic\Anthropic(
+                    key: $a['key'],
+                    model: $a['model'],
+                    parameters: $aParameters['parameters'] ?? [],
+                );
+                break;
+            case 'openai-responses':
+                $o = new NeuronAI\Providers\OpenAI\Responses\OpenAIResponses(
+                    key: $a['key'],
+                    model: $a['model'],
+                    parameters: $aParameters['parameters'] ?? [],
+                    strict_response: false
+                );
+                break;
+            case 'openai-embeddings':
+                $o = new NeuronAI\RAG\Embeddings\OpenAIEmbeddingsProvider(
+                    key: $a['key'],
+                    model: $a['model'],
+                    dimensions: !empty($aParameters['dimensions']) ? (int)$aParameters['dimensions'] : 1024
+                );
+                break;
+            default:
+                bx_log('sys_agents', "Model type {$a['type']} is not supported", BX_LOG_ERR);
+                throw new Exception("Model type {$a['type']} is not supported");
+        }
+
+        $GLOBALS['bxDolClasses'][__CLASS__ . '_Model_' . $iId] = $o;
+
+        return $o;
+    }
+
+    public static function getVectorStoreInstance(int $iId): NeuronAI\RAG\VectorStore\VectorStoreInterface 
+    {
+        if (isset($GLOBALS['bxDolClasses'][__CLASS__ . '_VectorStore_' . $iId]))
+            return $GLOBALS['bxDolClasses'][__CLASS__ . '_VectorStore_' . $iId];
+
+        $a = BxDolAIQuery::getVectorStoreObject($iId);
+        if (!$a) {
+            bx_log('sys_agents', "Vector store with id {$iId} not found", BX_LOG_ERR);
+            throw new Exception("Vector store with id {$iId} not found");
+        }
+
+        $aParameters = !empty($a['params']) ? json_decode($a['params'], true) : [];
+
+        switch($a['type']) {
+            // TODO: add more vector store types
+            case 'file':
+                $o = new NeuronAI\RAG\VectorStore\FileVectorStore(
+                    directory: BX_DIRECTORY_STORAGE . 'vector_stores/' . $iId,
+                    topK: $a['topk'],
+                );
+                break;
+            default:
+                bx_log('sys_agents', "Vector store type {$a['type']} is not supported", BX_LOG_ERR);
+                throw new Exception("Vector store type {$a['type']} is not supported");
+        }
+
+        $GLOBALS['bxDolClasses'][__CLASS__ . '_VectorStore_' . $iId] = $o;
+
+        return $o;
+    }
+
     public static function callHelper($mixedHelper, $sMessage)
     {
         $oAI = BxDolAI::getInstance();

inc/classes/BxDolAIQuery.php

@@ -25,6 +25,17 @@ class BxDolAIQuery extends BxDolDb
         return $aModel;
     }
 
+    static public function getVectorStoreObject($iId)
+    {
+        $oDb = BxDolDb::getInstance();
+
+        $aVectorStore = $oDb->getRow("SELECT * FROM `sys_agents_vector_store` WHERE `id` = :id", ['id' => $iId]);
+        if(!$aVectorStore || !is_array($aVectorStore))
+            return false;
+
+        return $aVectorStore;
+    }
+
     static public function getProviderObject($iId)
     {
         $oDb = BxDolDb::getInstance();
@@ -841,10 +852,16 @@ class BxDolAIQuery extends BxDolDb
             'ts' => time()]) > 0;
     }
 
+    public function getVectorStoreDataById (int $iId): mixed
+    {
+        $sQuery = "SELECT * FROM `sys_agents_vector_store_data` WHERE `id` = :id";
+        return $this->getRow($sQuery, ['id' => $iId]);
+    }
+
     static public function getVectorStorePendingData (int $iLimit = 1): mixed
     {
         $oDb = BxDolDb::getInstance();
-        $sQuery = "SELECT * FROM `sys_agents_vector_store_data` ORDER BY `added` ASC LIMIT :limit";
+        $sQuery = "SELECT * FROM `sys_agents_vector_store_data` WHERE `status` = 'pending' ORDER BY `added` ASC LIMIT :limit";
         return $oDb->getAll($sQuery, ['limit' => $iLimit]);
     }
 

inc/classes/BxDolCronAgentsVectorStore.php

@@ -11,7 +11,7 @@ class BxDolCronAgentsVectorStore extends BxDolCron
 {
     public function processing()
     {
-        // TODO:
+        BxBaseStudioAgentsVectorStoreData::processPendingData();
     }
 }
 

install/sql/system.sql

@@ -2639,7 +2639,8 @@ INSERT INTO `sys_cron_jobs` (`name`, `time`, `class`, `file`, `service_call`) VA
 ('sys_queue_email', '* * * * *', 'BxDolCronQueueEmail', 'inc/classes/BxDolCronQueueEmail.php', ''),
 ('sys_queue_push', '* * * * *', 'BxDolCronQueuePush', 'inc/classes/BxDolCronQueuePush.php', ''),
 ('sys_audit_clean', '* * * * *', 'BxDolCronAudit', 'inc/classes/BxDolCronAudit.php', ''),
-('sys_background_jobs', '* * * * *', 'BxDolCronBackgroundJobs', 'inc/classes/BxDolCronBackgroundJobs.php', '');
+('sys_background_jobs', '* * * * *', 'BxDolCronBackgroundJobs', 'inc/classes/BxDolCronBackgroundJobs.php', ''),
+('sys_agents_vector_store', '* * * * *', 'BxDolCronAgentsVectorStore', 'inc/classes/BxDolCronAgentsVectorStore.php', '');
 
 -- --------------------------------------------------------
 
@@ -6725,7 +6726,7 @@ CREATE TABLE `sys_agents_models` (
   `model` varchar(64) NOT NULL,
   `title` varchar(64) NOT NULL DEFAULT '',
   `docs` text NOT NULL,
-  `key` varchar(64) NOT NULL DEFAULT '',
+  `key` varchar(255) NOT NULL DEFAULT '',
   `params` text NOT NULL,
   `params_user` text DEFAULT NULL,
   `capabilities` enum('chatllm','chatvlm','embeddings') NOT NULL DEFAULT 'chatllm',
@@ -7082,7 +7083,7 @@ CREATE TABLE IF NOT EXISTS `sys_agents_vector_store_data` (
   `metadata` text NOT NULL DEFAULT '',
   `settings` varchar(255) NOT NULL DEFAULT '',
   `content` longtext NOT NULL DEFAULT '',
-  `status` enum('pending', 'processing','ready') NOT NULL DEFAULT 'pending',  
+  `status` enum('pending', 'processing','ready','error') NOT NULL DEFAULT 'pending',  
   `added` int(11) NOT NULL DEFAULT 0,
   PRIMARY KEY (`id`)
 );

studio/template/scripts/BxBaseStudioAgentsModels.php

@@ -97,7 +97,7 @@ class BxBaseStudioAgentsModels extends BxDolStudioAgentsModels
 
         $oParsedown = new Parsedown();
         $oParsedown->setSafeMode(false);
-        $sDocs = $oParsedown->text($aModel['docs']);        
+        $sDocs = $oParsedown->text($aModel['docs']);
 
         $aForm = array(
             'form_attrs' => array(

studio/template/scripts/BxBaseStudioAgentsVectorStoreData.php

@@ -19,6 +19,11 @@ class BxBaseStudioAgentsVectorStoreData extends BxDolStudioAgentsVectorStoreData
     protected function _delete ($mixedId)
     {
         // TODO: delete files from vector store
+        $r = $this->getVectorStoreDataById($mixedId);
+        $oVectorStore = BxDolAI::getVectorStoreInstance($r['vector_store_id']);
+        if ($oVectorStore) {
+            $oVectorStore->deleteBySource($r['vector_store_id'], $r['id']);
+        }
 
         $mixedResult = parent::_delete($mixedId);
         return $mixedResult;
@@ -32,13 +37,80 @@ class BxBaseStudioAgentsVectorStoreData extends BxDolStudioAgentsVectorStoreData
 
     protected function _getCellStatus ($mixedValue, $sKey, $aField, $aRow) 
     {
-        $mixedValue = $mixedValue == 'processing' ? '<span title="Processing">🕚</span>' : '<span title="Completed">✅</span>';
+        switch ($mixedValue) {
+            case 'pending':
+                $mixedValue = '<span title="Pending">⏳</span>';
+                break;
+            case 'processing':
+                $mixedValue = '<span title="Processing">⚙️</span>';
+                break;
+            case 'ready':
+                $mixedValue = '<span title="Ready">✅</span>';
+                break;
+            case 'error':
+                $mixedValue = '<span title="Error">❌</span>';
+                break;
+        }        
         return parent::_getCellDefault ($mixedValue, $sKey, $aField, $aRow);
     } 
 
     static public function processPendingData()
-    {
-        // TODO: process pending data, e.g. add to vector store, update status, etc.
+    { 
+        $iLimit = 5; // TODO: add to settings
+        for ($i = 0; $i < $iLimit; $i++) {
+            $a = BxDolAiQuery::getVectorStorePendingData(1);
+            if (!$a)
+                break;
+            foreach ($a as $r) {
+                BxDolAiQuery::updateVectorStoreDataStatus($r['id'], 'processing');
+
+                $aVectorStore = BxDolAiQuery::getVectorStoreObject($r['vector_store_id']);
+                $oVectorStore = BxDolAI::getVectorStoreInstance($aVectorStore['id']);
+                $oEmbedder = BxDolAI::getModelInstance($aVectorStore['embedding_provider_id']);
+                $sError = '';
+
+                if ($oEmbedder && $oVectorStore) {
+                    try {
+                        $aSettings = $r['settings'] ? json_decode($r['settings'], true) : [];
+                        $aMetadata = $r['metadata'] ? json_decode($r['metadata'], true) : [];
+
+                        $documents = NeuronAI\RAG\DataLoader\StringDataLoader::for($r['content'])
+                            ->withSplitter(
+                            new NeuronAI\RAG\Splitter\DelimiterTextSplitter(
+                                    maxLength: isset($aSettings['chunk_size']) ? $aSettings['chunk_size'] : 512,
+                                    separator: isset($aSettings['delimiter']) ? $aSettings['delimiter'] : '.',
+                                    wordOverlap: isset($aSettings['overlap']) ? $aSettings['overlap'] : 2
+                                )
+                            )
+                            ->getDocuments(); 
+                        
+                        foreach ($documents as $document) {
+                            $document->sourceType = $r['vector_store_id'];
+                            $document->sourceName = $r['id'];
+                            if (!empty($aMetadata))
+                                $document->metadata = $aMetadata;
+                        }
+
+                        $oVectorStore->addDocuments(
+                            $oEmbedder->embedDocuments($documents)
+                        );
+                    } 
+                    catch (Exception $e) {
+                        $sError = "Error process vector store data ({$r['id']}): " . $e->getMessage();
+                    }
+                    if (!$sError)
+                        BxDolAiQuery::updateVectorStoreDataStatus($r['id'], 'ready');
+                } 
+                else {
+                    $sError = "Can't process vector store data with id {$r['id']}, missing embedder({$aVectorStore['embedding_provider_id']}) or vector store({$aVectorStore['id']}) instance";
+                }
+
+                if ($sError) {
+                    bx_log('sys_agents', $sError, BX_LOG_ERR);
+                    BxDolAiQuery::updateVectorStoreDataStatus($r['id'], 'error');
+                }
+            }
+        }
     }
 }