Merge branch 'master' of https://github.com/unacms/UNA

inc/classes/BxDolAI.php

@@ -70,6 +70,104 @@ class BxDolAI extends BxDolFactory implements iBxDolSingleton
         return $GLOBALS['bxDolClasses'][__CLASS__];
     }
 
+    public static function getAiProviderInstance(int $iId):NeuronAI\Providers\AIProviderInterface
+    {
+        return self::getInstance()->getAiProviderInstance($iId);
+    }   
+
+    public static function getAiEmbeddingsProviderInstance(int $iId):NeuronAI\RAG\Embeddings\EmbeddingsProviderInterface
+    {
+        return self::getInstance()->getAiProviderInstance($iId);
+    }   
+
+    public static function getModelInstance(int $iId): NeuronAI\Providers\AIProviderInterface | NeuronAI\RAG\Embeddings\EmbeddingsProviderInterface
+    {
+        if (isset($GLOBALS['bxDolClasses'][__CLASS__ . '_Model_' . $iId]))
+            return $GLOBALS['bxDolClasses'][__CLASS__ . '_Model_' . $iId];
+
+        $aProvidersWithKey = ['anthropic'];
+        $a = BxDolAIQuery::getModelObject($iId);
+        if (!$a) {
+            bx_log('sys_agents', "Agent AI Model with id {$iId} not found", BX_LOG_ERR);
+            throw new Exception("Agent AI Model with id {$iId} not found");
+        }
+        if (in_array($a['type'], $aProvidersWithKey) && empty($a['key'])) {
+            bx_log('sys_agents', "Model with id {$iId} has empty key, can't be used", BX_LOG_ERR);
+            throw new Exception("Model with id {$iId} has empty key, can't be used");
+        }
+
+        if (!$a['active']) {
+            bx_log('sys_agents', "Model with id {$iId} is not active, can't be used", BX_LOG_ERR);
+            throw new Exception("Model with id {$iId} is not active, can't be used");
+        }
+
+        $aParameters = !empty($a['params']) ? json_decode($a['params'], true) : [];
+
+        switch($a['type']) {
+            // TODO: add more providers
+            case 'anthropic':
+                $o = new NeuronAI\Providers\Anthropic\Anthropic(
+                    key: $a['key'],
+                    model: $a['model'],
+                    parameters: $aParameters['parameters'] ?? [],
+                );
+                break;
+            case 'openai-responses':
+                $o = new NeuronAI\Providers\OpenAI\Responses\OpenAIResponses(
+                    key: $a['key'],
+                    model: $a['model'],
+                    parameters: $aParameters['parameters'] ?? [],
+                    strict_response: false
+                );
+                break;
+            case 'openai-embeddings':
+                $o = new NeuronAI\RAG\Embeddings\OpenAIEmbeddingsProvider(
+                    key: $a['key'],
+                    model: $a['model'],
+                    dimensions: !empty($aParameters['dimensions']) ? (int)$aParameters['dimensions'] : 1024
+                );
+                break;
+            default:
+                bx_log('sys_agents', "Model type {$a['type']} is not supported", BX_LOG_ERR);
+                throw new Exception("Model type {$a['type']} is not supported");
+        }
+
+        $GLOBALS['bxDolClasses'][__CLASS__ . '_Model_' . $iId] = $o;
+
+        return $o;
+    }
+
+    public static function getVectorStoreInstance(int $iId): NeuronAI\RAG\VectorStore\VectorStoreInterface 
+    {
+        if (isset($GLOBALS['bxDolClasses'][__CLASS__ . '_VectorStore_' . $iId]))
+            return $GLOBALS['bxDolClasses'][__CLASS__ . '_VectorStore_' . $iId];
+
+        $a = BxDolAIQuery::getVectorStoreObject($iId);
+        if (!$a) {
+            bx_log('sys_agents', "Vector store with id {$iId} not found", BX_LOG_ERR);
+            throw new Exception("Vector store with id {$iId} not found");
+        }
+
+        $aParameters = !empty($a['params']) ? json_decode($a['params'], true) : [];
+
+        switch($a['type']) {
+            // TODO: add more vector store types
+            case 'file':
+                $o = new NeuronAI\RAG\VectorStore\FileVectorStore(
+                    directory: BX_DIRECTORY_STORAGE . 'vector_stores/' . $iId,
+                    topK: $a['topk'],
+                );
+                break;
+            default:
+                bx_log('sys_agents', "Vector store type {$a['type']} is not supported", BX_LOG_ERR);
+                throw new Exception("Vector store type {$a['type']} is not supported");
+        }
+
+        $GLOBALS['bxDolClasses'][__CLASS__ . '_VectorStore_' . $iId] = $o;
+
+        return $o;
+    }
+
     public static function callHelper($mixedHelper, $sMessage)
     {
         $oAI = BxDolAI::getInstance();
@@ -121,6 +219,8 @@ class BxDolAI extends BxDolFactory implements iBxDolSingleton
         $aParamsDb = ['sample' => 'all_pairs'];
         if(isset($aParams['active']))
             $aParamsDb['active'] = $aParams['active'] === true ? 1 : 0;
+        if(isset($aParams['capabilities']))
+            $aParamsDb['capabilities'] = $aParams['capabilities'];
 
         return $aModel = $this->_oDb->getModelsBy($aParamsDb);
     }

inc/classes/BxDolAIQuery.php

@@ -25,6 +25,17 @@ class BxDolAIQuery extends BxDolDb
         return $aModel;
     }
 
+    static public function getVectorStoreObject($iId)
+    {
+        $oDb = BxDolDb::getInstance();
+
+        $aVectorStore = $oDb->getRow("SELECT * FROM `sys_agents_vector_store` WHERE `id` = :id", ['id' => $iId]);
+        if(!$aVectorStore || !is_array($aVectorStore))
+            return false;
+
+        return $aVectorStore;
+    }
+
     static public function getProviderObject($iId)
     {
         $oDb = BxDolDb::getInstance();
@@ -108,6 +119,12 @@ class BxDolAIQuery extends BxDolDb
                     $sWhereClause .= " AND `active`=:active";
                 }
 
+                if(isset($aParams['capabilities'])) {
+                    $aMethod['params'][3]['capabilities'] = $aParams['capabilities'];
+
+                    $sWhereClause .= " AND `capabilities` = :capabilities";
+                }
+
                 break;
         }
 
@@ -835,10 +852,16 @@ class BxDolAIQuery extends BxDolDb
             'ts' => time()]) > 0;
     }
 
+    public function getVectorStoreDataById (int $iId): mixed
+    {
+        $sQuery = "SELECT * FROM `sys_agents_vector_store_data` WHERE `id` = :id";
+        return $this->getRow($sQuery, ['id' => $iId]);
+    }
+
     static public function getVectorStorePendingData (int $iLimit = 1): mixed
     {
         $oDb = BxDolDb::getInstance();
-        $sQuery = "SELECT * FROM `sys_agents_vector_store_data` ORDER BY `added` ASC LIMIT :limit";
+        $sQuery = "SELECT * FROM `sys_agents_vector_store_data` WHERE `status` = 'pending' ORDER BY `added` ASC LIMIT :limit";
         return $oDb->getAll($sQuery, ['limit' => $iLimit]);
     }
 

inc/classes/BxDolCronAgentsVectorStore.php

@@ -11,7 +11,7 @@ class BxDolCronAgentsVectorStore extends BxDolCron
 {
     public function processing()
     {
-        // TODO:
+        BxBaseStudioAgentsVectorStoreData::processPendingData();
     }
 }
 

install/sql/system.sql

@@ -2639,7 +2639,8 @@ INSERT INTO `sys_cron_jobs` (`name`, `time`, `class`, `file`, `service_call`) VA
 ('sys_queue_email', '* * * * *', 'BxDolCronQueueEmail', 'inc/classes/BxDolCronQueueEmail.php', ''),
 ('sys_queue_push', '* * * * *', 'BxDolCronQueuePush', 'inc/classes/BxDolCronQueuePush.php', ''),
 ('sys_audit_clean', '* * * * *', 'BxDolCronAudit', 'inc/classes/BxDolCronAudit.php', ''),
-('sys_background_jobs', '* * * * *', 'BxDolCronBackgroundJobs', 'inc/classes/BxDolCronBackgroundJobs.php', '');
+('sys_background_jobs', '* * * * *', 'BxDolCronBackgroundJobs', 'inc/classes/BxDolCronBackgroundJobs.php', ''),
+('sys_agents_vector_store', '* * * * *', 'BxDolCronAgentsVectorStore', 'inc/classes/BxDolCronAgentsVectorStore.php', '');
 
 -- --------------------------------------------------------
 
@@ -5785,16 +5786,15 @@ INSERT INTO `sys_grid_fields` (`object`, `name`, `title`, `width`, `translatable
 ('sys_studio_agents_vector_store', 'switcher', '_sys_active', '10%', 0, 0, '', '', 10),
 ('sys_studio_agents_vector_store', 'title', '_Title', '20%', 0, 0, '', '', 20),
 ('sys_studio_agents_vector_store', 'type', '_adm_form_txt_fields_type', '10%', 0, 0, '', '', 30),
-('sys_studio_agents_vector_store', 'topk', '_sys_agents_vector_store_txt_topk', '5%', 0, 0, '', '', 40),
+('sys_studio_agents_vector_store', 'embedding_provider_id', '_sys_agents_vector_store_txt_embedding_provider', '5%', 0, 0, '', '', 40),
 ('sys_studio_agents_vector_store', 'files_num', '_sys_agents_vector_store_txt_files_num', '5%', 0, 0, '', '', 50),
 ('sys_studio_agents_vector_store', 'actions', '', '25%', 0, 0, '', '', 60);
 
 INSERT INTO `sys_grid_actions` (`object`, `type`, `name`, `title`, `icon`, `icon_only`, `confirm`, `active`, `order`) VALUES
 ('sys_studio_agents_vector_store', 'single', 'add_data', '_sys_uploader_simple_attach_one_more_file', 'plus', 1, 0, 1, 10),
-('sys_studio_agents_vector_store', 'single', 'files', '_adm_lmi_cpt_files', 'folder', 1, 0, 1, 20),
-('sys_studio_agents_vector_store', 'single', 'edit', '_Edit', 'pencil-alt', 1, 0, 1, 30),
-('sys_studio_agents_vector_store', 'single', 'delete', '_Delete', 'remove', 1, 1, 1, 40),
-('sys_studio_agents_vector_store', 'single', 'duplicate', '_Duplicate', 'copy', 1, 0, 1, 50);
+('sys_studio_agents_vector_store', 'single', 'edit', '_Edit', 'pencil-alt', 1, 0, 1, 20),
+('sys_studio_agents_vector_store', 'single', 'delete', '_Delete', 'remove', 1, 1, 1, 30),
+('sys_studio_agents_vector_store', 'single', 'duplicate', '_Duplicate', 'copy', 1, 0, 1, 40);
 
 -- GRID: Agents Vector Store Data
 INSERT INTO `sys_objects_grid` (`object`, `source_type`, `source`, `table`, `field_id`, `field_order`, `field_active`, `paginate_url`, `paginate_per_page`, `paginate_simple`, `paginate_get_start`, `paginate_get_per_page`, `filter_fields`, `filter_fields_translatable`, `filter_mode`, `sorting_fields`, `sorting_fields_translatable`, `visible_for_levels`, `responsive`, `show_total_count`, `override_class_name`, `override_class_file`) VALUES
@@ -6726,7 +6726,7 @@ CREATE TABLE `sys_agents_models` (
   `model` varchar(64) NOT NULL,
   `title` varchar(64) NOT NULL DEFAULT '',
   `docs` text NOT NULL,
-  `key` varchar(64) NOT NULL DEFAULT '',
+  `key` varchar(255) NOT NULL DEFAULT '',
   `params` text NOT NULL,
   `params_user` text DEFAULT NULL,
   `capabilities` enum('chatllm','chatvlm','embeddings') NOT NULL DEFAULT 'chatllm',
@@ -7083,15 +7083,17 @@ CREATE TABLE IF NOT EXISTS `sys_agents_vector_store_data` (
   `metadata` text NOT NULL DEFAULT '',
   `settings` varchar(255) NOT NULL DEFAULT '',
   `content` longtext NOT NULL DEFAULT '',
-  `status` enum('pending', 'processing','ready') NOT NULL DEFAULT 'pending',  
+  `status` enum('pending', 'processing','ready','error') NOT NULL DEFAULT 'pending',  
   `added` int(11) NOT NULL DEFAULT 0,
   PRIMARY KEY (`id`)
 );
 
 CREATE TABLE IF NOT EXISTS `sys_agents_vector_store` (
   `id` int(11) NOT NULL AUTO_INCREMENT,  
+  `embedding_provider_id` int(11) NOT NULL,
   `type` varchar(128) NOT NULL,
   `title` varchar(255) NOT NULL DEFAULT '',
+  `docs` text DEFAULT NULL,
   `topk` tinyint(4) DEFAULT 4,
   `params` text DEFAULT NULL,
   `params_user` text DEFAULT NULL,

studio/template/scripts/BxBaseStudioAgentsModels.php

@@ -56,7 +56,7 @@ class BxBaseStudioAgentsModels extends BxDolStudioAgentsModels
         $oForm->initChecker();
 
         if($oForm->isSubmittedAndValid()) {
-            $aValsToAdd = [];
+            $aValsToAdd = ['changed' => time()];
             
             if($oForm->update($iId, $aValsToAdd) !== false)
                 $aRes = ['grid' => $this->getCode(false), 'blink' => $iId];

studio/template/scripts/BxBaseStudioAgentsVectorStore.php

@@ -90,7 +90,7 @@ class BxBaseStudioAgentsVectorStore extends BxDolStudioAgentsVectorStore
                     $iFilesAdded++;
                 }
             }
-            return echoJson(['msg' => _t('_sys_agents_vector_store_data_queued', $iFilesAdded) . ' / num:' . $iFilesAdded]);
+            return echoJson(['msg' => _t('_sys_agents_vector_store_data_queued', $iFilesAdded)]);
 
             if($oForm->add($iId) === false)
                 return echoJson(['msg' => _t('_sys_txt_error_occured')]);
@@ -168,6 +168,16 @@ class BxBaseStudioAgentsVectorStore extends BxDolStudioAgentsVectorStore
         $data = json_decode($aVectorStore['params_user'], true);
         $aVectorStore['params_user'] = json_encode($data, JSON_PRETTY_PRINT);
 
+        $oParsedown = new Parsedown();
+        $oParsedown->setSafeMode(false);
+        $sDocs = $oParsedown->text($aVectorStore['docs']);
+
+        $aEmbeddingProviders = ['' => _t('_sys_please_select')];
+        $a = BxDolAI::getInstance()->getModels(['capabilities' => 'embeddings', 'active' => true]);
+        foreach ($a as $k => $v) {
+            $aEmbeddingProviders[$k] = $v;
+        }
+
         return [
             'form_attrs' => [
                 'id' => 'bx_std_agents_helpers_' . $sAction,
@@ -182,10 +192,28 @@ class BxBaseStudioAgentsVectorStore extends BxDolStudioAgentsVectorStore
                 ),
             ),
             'inputs' => [
+                'docs' => [
+                    'type' => 'custom',
+                    'name' => 'docs',
+                    'caption' => '',
+                    'content' => $sDocs,
+                ],
+                'embedding_provider_id' => [
+                    'type' => 'select',
+                    'name' => 'embedding_provider_id',
+                    'caption' => '_sys_agents_vector_store_field_embedding_provider_id',
+                    'info' => '_sys_agents_vector_store_field_embedding_provider_id_info',
+                    'value' => !empty($aVectorStore['embedding_provider_id']) ? $aVectorStore['embedding_provider_id'] : '',
+                    'values' => $aEmbeddingProviders,
+                    'db' => [
+                        'pass' => 'Int'
+                    ]
+                ],
                 'topk' => [
                     'type' => 'text',
                     'name' => 'topk',
                     'caption' => '_sys_agents_vector_store_field_topk',
+                    'info' => '_sys_agents_vector_store_field_topk_info',
                     'required' => true,
                     'value' => !empty($aVectorStore['topk']) ? $aVectorStore['topk'] : '',
                     'db' => [
@@ -196,6 +224,7 @@ class BxBaseStudioAgentsVectorStore extends BxDolStudioAgentsVectorStore
                     'type' => 'textarea',
                     'name' => 'params_user',
                     'caption' => '_sys_agents_vector_store_field_params',
+                    'info' => '_sys_agents_vector_store_field_params_info',
                     'required' => false,
                     'value' => !empty($aVectorStore['params_user']) ? $aVectorStore['params_user'] : '',
                     'checker' => [
@@ -320,10 +349,19 @@ class BxBaseStudioAgentsVectorStore extends BxDolStudioAgentsVectorStore
         return $html->getText();
     }
 
+    protected function _getCellEmbeddingProviderId($mixedValue, $sKey, $aField, $aRow) 
+    {
+        $aProvider = $this->_oDb->getModelsBy(['sample' => 'id', 'id' => $mixedValue]);
+        $s = !empty($aProvider) ? $aProvider['title'] : _t('_undefined');
+        return parent::_getCellDefault($s, $sKey, $aField, $aRow);
+    }
+    
     protected function _getCellFilesNum($mixedValue, $sKey, $aField, $aRow)
     {
-        $mixedValue = $this->_oDb->getVectorStoreDataNum($aRow['id']);
-        return parent::_getCellDefault($mixedValue, $sKey, $aField, $aRow);
+        $s = $this->_oDb->getVectorStoreDataNum($aRow['id']);
+        if ((int)$s !== 0)
+            $s = '<a href="javascript:void(0)" onclick="glGrids.' . $this->_sObject . '.actionWithId (' . $aRow['id'] . ', \'files\', {}, \'\', false, 0);">' . $s . '</a>';
+        return parent::_getCellDefault($s, $sKey, $aField, $aRow);
     }
     
     protected function _getActionDelete ($sType, $sKey, $a, $isSmall = false, $isDisabled = false, $aRow = array())

studio/template/scripts/BxBaseStudioAgentsVectorStoreData.php

@@ -18,7 +18,11 @@ class BxBaseStudioAgentsVectorStoreData extends BxDolStudioAgentsVectorStoreData
 
     protected function _delete ($mixedId)
     {
-        // TODO: delete files from vector store
+        $r = $this->_oDb->getVectorStoreDataById($mixedId);
+        $oVectorStore = BxDolAI::getVectorStoreInstance($r['vector_store_id']);
+        if ($oVectorStore) {
+            $oVectorStore->deleteBySource($r['vector_store_id'], $r['id']);
+        }
 
         $mixedResult = parent::_delete($mixedId);
         return $mixedResult;
@@ -32,13 +36,80 @@ class BxBaseStudioAgentsVectorStoreData extends BxDolStudioAgentsVectorStoreData
 
     protected function _getCellStatus ($mixedValue, $sKey, $aField, $aRow) 
     {
-        $mixedValue = $mixedValue == 'processing' ? '<span title="Processing">🕚</span>' : '<span title="Completed">✅</span>';
+        switch ($mixedValue) {
+            case 'pending':
+                $mixedValue = '<span title="Pending">⏳</span>';
+                break;
+            case 'processing':
+                $mixedValue = '<span title="Processing">⚙️</span>';
+                break;
+            case 'ready':
+                $mixedValue = '<span title="Ready">✅</span>';
+                break;
+            case 'error':
+                $mixedValue = '<span title="Error">❌</span>';
+                break;
+        }        
         return parent::_getCellDefault ($mixedValue, $sKey, $aField, $aRow);
     } 
 
     static public function processPendingData()
-    {
-        // TODO: process pending data, e.g. add to vector store, update status, etc.
+    { 
+        $iLimit = 5; // TODO: add to settings
+        for ($i = 0; $i < $iLimit; $i++) {
+            $a = BxDolAiQuery::getVectorStorePendingData(1);
+            if (!$a)
+                break;
+            foreach ($a as $r) {
+                BxDolAiQuery::updateVectorStoreDataStatus($r['id'], 'processing');
+
+                $aVectorStore = BxDolAiQuery::getVectorStoreObject($r['vector_store_id']);
+                $oVectorStore = BxDolAI::getVectorStoreInstance($aVectorStore['id']);
+                $oEmbedder = BxDolAI::getModelInstance($aVectorStore['embedding_provider_id']);
+                $sError = '';
+
+                if ($oEmbedder && $oVectorStore) {
+                    try {
+                        $aSettings = $r['settings'] ? json_decode($r['settings'], true) : [];
+                        $aMetadata = $r['metadata'] ? json_decode($r['metadata'], true) : [];
+
+                        $documents = NeuronAI\RAG\DataLoader\StringDataLoader::for($r['content'])
+                            ->withSplitter(
+                            new NeuronAI\RAG\Splitter\DelimiterTextSplitter(
+                                    maxLength: isset($aSettings['chunk_size']) ? $aSettings['chunk_size'] : 512,
+                                    separator: isset($aSettings['delimiter']) ? $aSettings['delimiter'] : '.',
+                                    wordOverlap: isset($aSettings['overlap']) ? $aSettings['overlap'] : 2
+                                )
+                            )
+                            ->getDocuments(); 
+                        
+                        foreach ($documents as $document) {
+                            $document->sourceType = $r['vector_store_id'];
+                            $document->sourceName = $r['id'];
+                            if (!empty($aMetadata))
+                                $document->metadata = $aMetadata;
+                        }
+
+                        $oVectorStore->addDocuments(
+                            $oEmbedder->embedDocuments($documents)
+                        );
+                    } 
+                    catch (Exception $e) {
+                        $sError = "Error process vector store data ({$r['id']}): " . $e->getMessage();
+                    }
+                    if (!$sError)
+                        BxDolAiQuery::updateVectorStoreDataStatus($r['id'], 'ready');
+                } 
+                else {
+                    $sError = "Can't process vector store data with id {$r['id']}, missing embedder({$aVectorStore['embedding_provider_id']}) or vector store({$aVectorStore['id']}) instance";
+                }
+
+                if ($sError) {
+                    bx_log('sys_agents', $sError, BX_LOG_ERR);
+                    BxDolAiQuery::updateVectorStoreDataStatus($r['id'], 'error');
+                }
+            }
+        }
     }
 }