Merge branch 'master' of https://github.com/unacms/UNA

composer.json

@@ -104,7 +104,8 @@
         "spatie/fork": "^1.2",
         "elasticsearch/elasticsearch": "^9.3",
         "opensearch-project/opensearch-php": "^2.5",
-        "typesense/typesense-php": "*"
+        "typesense/typesense-php": "*",
+        "html2text/html2text": "^4.3"
     },
     "scripts": {
         "post-update-cmd": [

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "9bcab32a46aa81ba1683add09894dd21",
+    "content-hash": "c9e95235742656e9fab6db18b8edf02e",
     "packages": [
         {
             "name": "achingbrain/php5-akismet",
@@ -1212,6 +1212,44 @@
             ],
             "time": "2025-08-23T21:21:41+00:00"
         },
+        {
+            "name": "html2text/html2text",
+            "version": "4.3.2",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/mtibben/html2text.git",
+                "reference": "3b443cbe302b52eb5806a21a9dbd79524203970a"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/mtibben/html2text/zipball/3b443cbe302b52eb5806a21a9dbd79524203970a",
+                "reference": "3b443cbe302b52eb5806a21a9dbd79524203970a",
+                "shasum": ""
+            },
+            "require-dev": {
+                "phpunit/phpunit": "~4|^9.0"
+            },
+            "suggest": {
+                "ext-mbstring": "For best performance",
+                "symfony/polyfill-mbstring": "If you can't install ext-mbstring"
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Html2Text\\": "src/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "GPL-2.0-or-later"
+            ],
+            "description": "Converts HTML to formatted plain text",
+            "support": {
+                "issues": "https://github.com/mtibben/html2text/issues",
+                "source": "https://github.com/mtibben/html2text/tree/4.3.2"
+            },
+            "time": "2024-08-20T02:43:29+00:00"
+        },
         {
             "name": "inspector-apm/inspector-php",
             "version": "3.16.15",

inc/classes/BxDolAIQuery.php

@@ -800,6 +800,30 @@ class BxDolAIQuery extends BxDolDb
         $sQuery = "SELECT * FROM `sys_agents_vector_store` WHERE `id` = :id";
         return $this->getRow($sQuery, ['id' => $iId]); 
     }
+
+    public function addVectorStoreData (int $iVectorStoreId, string $sType, string $sName, int $iFileSize, string $sMetadata, string $sSettings, string $sContent): bool
+    {
+        $sQuery = "INSERT INTO `sys_agents_vector_store_data` SET  
+            `vector_store_id` = :vector_store_id,
+            `type` = :type, 
+            `name` = :name, 
+            `size` = :size,
+            `metadata` = :metadata, 
+            `settings` = :settings, 
+            `content` = :content,
+            `added` = :ts
+        ";
+        return $this->query($sQuery, [
+            'vector_store_id' => $iVectorStoreId,
+            'type' => $sType, 
+            'name' => $sName, 
+            'size' => $iFileSize,
+            'metadata' => $sMetadata, 
+            'settings' => $sSettings, 
+            'content' => $sContent,
+            'ts' => time()]) > 0;
+    }
+
 }
 
 /** @} */

inc/classes/BxDolCronAgentsVectorStore.php

@@ -0,0 +1,18 @@
+<?php defined('BX_DOL') or die('hack attempt');
+/**
+* Copyright (c) UNA, Inc - https://una.io
+* MIT License - https://opensource.org/licenses/MIT
+*
+* @defgroup    UnaCore UNA Core
+* @{
+*/
+
+class BxDolCronAgentsVectorStore extends BxDolCron
+{
+    public function processing()
+    {
+        // TODO:
+    }
+}
+
+/** @} */

inc/utils.inc.php

@@ -1297,6 +1297,17 @@ function return_bytes($val)
     return $val;
 }
 
+/**
+ * Format bytes to human-readable format
+ * @param $iBytes - number of bytes
+ * @param $iPrecision - decimal places (default 1)
+ * @return formatted string like "2.5M", "512K", "1024B"
+ */
+function bx_format_bytes($iBytes)
+{
+    return _t_format_size($iBytes);
+}
+
 // Generate Random Password
 function genRndPwd($iLength = 8, $bSpecialCharacters = true)
 {

install/sql/system.sql

@@ -5791,10 +5791,25 @@ INSERT INTO `sys_grid_fields` (`object`, `name`, `title`, `width`, `translatable
 ('sys_studio_agents_vector_store', 'actions', '', '25%', 0, 0, '', '', 9);
 
 INSERT INTO `sys_grid_actions` (`object`, `type`, `name`, `title`, `icon`, `icon_only`, `confirm`, `active`, `order`) VALUES
-('sys_studio_agents_vector_store', 'single', 'edit', '_Edit', 'pencil-alt', 1, 0, 1, 1),
-('sys_studio_agents_vector_store', 'single', 'delete', '_Delete', 'remove', 1, 1, 1, 2),
-('sys_studio_agents_vector_store', 'single', 'duplicate', '_Duplicate', 'copy', 1, 1, 1, 3);
+('sys_studio_agents_vector_store', 'single', 'add_data', '_sys_uploader_simple_attach_one_more_file', 'plus', 1, 0, 1, 10),
+('sys_studio_agents_vector_store', 'single', 'files', '_adm_lmi_cpt_files', 'folder', 1, 0, 1, 20),
+('sys_studio_agents_vector_store', 'single', 'edit', '_Edit', 'pencil-alt', 1, 0, 1, 30),
+('sys_studio_agents_vector_store', 'single', 'delete', '_Delete', 'remove', 1, 1, 1, 40),
+('sys_studio_agents_vector_store', 'single', 'duplicate', '_Duplicate', 'copy', 1, 1, 1, 50);
 
+-- GRID: Agents Vector Store Data
+INSERT INTO `sys_objects_grid` (`object`, `source_type`, `source`, `table`, `field_id`, `field_order`, `field_active`, `paginate_url`, `paginate_per_page`, `paginate_simple`, `paginate_get_start`, `paginate_get_per_page`, `filter_fields`, `filter_fields_translatable`, `filter_mode`, `sorting_fields`, `sorting_fields_translatable`, `visible_for_levels`, `responsive`, `show_total_count`, `override_class_name`, `override_class_file`) VALUES
+('sys_studio_agents_vector_store_data', 'Sql', 'SELECT * FROM `sys_agents_vector_store_data` WHERE `vector_store_id` = ''{vector_store_id}'' ', 'sys_agents_vector_store_data', 'id', 'id', '', '', 10, NULL, 'start', '', 'name', '', 'like', '', '', 2147483647, 1, 1, 'BxTemplStudioAgentsVectorStoreData', '');
+
+INSERT INTO `sys_grid_fields` (`object`, `name`, `title`, `width`, `translatable`, `chars_limit`, `params`, `hidden_on`, `order`) VALUES
+('sys_studio_agents_vector_store_data', 'checkbox', '', '2%', 0, 0, '', '', 10),
+('sys_studio_agents_vector_store_data', 'name', '_Name', '70%', 0, 0, '', '', 20),
+('sys_studio_agents_vector_store_data', 'size', '_sys_agents_assistants_files_txt_size', '18%', 0, 0, '', '', 30),
+('sys_studio_agents_vector_store_data', 'status', '_sys_status', '10%', 0, 0, '', '', 40);
+
+
+INSERT INTO `sys_grid_actions` (`object`, `type`, `name`, `title`, `icon`, `icon_only`, `confirm`, `active`, `order`) VALUES
+('sys_studio_agents_vector_store_data', 'bulk', 'delete', '_Delete', '', 0, 1, 1, 10);
 
 -- --------------------------------------------------------
 
@@ -6937,6 +6952,20 @@ CREATE TABLE IF NOT EXISTS `sys_agents_assistants_chats_files` (
   UNIQUE KEY `remote_id` (`remote_id`)
 );
 
+CREATE TABLE IF NOT EXISTS `sys_agents_vector_store_data` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,  
+  `vector_store_id` int(11) NOT NULL,
+  `type` varchar(128) NOT NULL,
+  `name` varchar(255) NOT NULL DEFAULT '',
+  `size` int(11) NOT NULL DEFAULT 0,
+  `metadata` text NOT NULL DEFAULT '',
+  `settings` varchar(255) NOT NULL DEFAULT '',
+  `content` longtext NOT NULL DEFAULT '',
+  `status` enum('pending', 'processing','ready') NOT NULL DEFAULT 'pending',
+  `added` int(11) NOT NULL DEFAULT 0,
+  PRIMARY KEY (`id`)
+);
+
 CREATE TABLE IF NOT EXISTS `sys_agents_vector_store` (
   `id` int(11) NOT NULL AUTO_INCREMENT,  
   `type` varchar(128) NOT NULL,

modules/boonex/profiler/classes/BxProfiler.php

@@ -712,12 +712,7 @@ class BxProfiler extends BxDol
 
     function _formatBytes ($i)
     {
-        if ($i > 1024*1024)
-            return round($i/1024/1024, 1) . 'M';
-        elseif ($i > 1024)
-            return round($i/1024, 1) . 'K';
-        else
-            return $i . 'B';
+        return bx_format_bytes($i);
     }
 
     function _formatKb ($i)

studio/classes/BxDolStudioAgentsVectorStoreData.php

@@ -0,0 +1,24 @@
+<?php defined('BX_DOL') or die('hack attempt');
+/**
+ * Copyright (c) UNA, Inc - https://una.io
+ * MIT License - https://opensource.org/licenses/MIT
+ *
+ * @defgroup    UnaStudio UNA Studio
+ * @{
+ */
+
+class BxDolStudioAgentsVectorStoreData extends BxTemplStudioGridAgents
+{
+    protected $_oDb;
+
+    public function __construct ($aOptions, $oTemplate = false)
+    {
+        parent::__construct ($aOptions, $oTemplate);
+
+        $this->_sDefaultSortingOrder = 'DESC';
+
+        $this->_oDb = new BxDolStudioAgentsQuery();
+    }
+}
+
+/** @} */

studio/template/agents_popup_grid.html

@@ -0,0 +1,5 @@
+<div class="bx-agents-automator-form">
+    <div class="bx-agents-automator-form-cnt">
+        __grid__
+    </div>
+</div>

studio/template/scripts/BxBaseStudioAgentsVectorStore.php

@@ -12,6 +12,11 @@ class BxBaseStudioAgentsVectorStore extends BxDolStudioAgentsVectorStore
 {
     protected $_sUrlPage;
     protected $_sFieldName;
+    protected $_aDataExt = [
+        'txt' => 'getFileContentPlain',
+        'md' => 'getFileContentPlain',
+        'html' => 'getFileContentHtml',
+    ];
 
     public function __construct ($aOptions, $oTemplate = false)
     {
@@ -26,6 +31,64 @@ class BxBaseStudioAgentsVectorStore extends BxDolStudioAgentsVectorStore
     {
         return 'oBxDolStudioPageAgents';
     }
+    
+    public function performActionAddData()
+    {
+        $sAction = 'add_data';
+
+        $iId = $this->_getId();
+        $aVectorStore = $this->_oDb->getVectorStoreById($iId);
+
+        $aForm = $this->_getFormAddData($sAction, $aVectorStore);
+        $oForm = new BxTemplFormView($aForm);
+        $oForm->initChecker();
+
+        if($oForm->isSubmittedAndValid()) {
+            $iFilesAdded = 0;
+            foreach ($_FILES['files']['tmp_name'] as $key => $tmp_name) {
+                $sFileName = $_FILES['files']['name'][$key];
+                $sFileType = $_FILES['files']['type'][$key];
+                $sFileSize = $_FILES['files']['size'][$key];
+                $sFileTmpName = $_FILES['files']['tmp_name'][$key];
+                $sMetadata = $oForm->getCleanValue('metadata');
+                $iVectorStoreId = $oForm->getCleanValue('vector_store_id');
+                $sSettings = json_encode([
+                    'chunk_size' => $oForm->getCleanValue('chunk_size'),
+                    'delimeter' => trim($oForm->getCleanValue('delimeter')),
+                    'overlap' => $oForm->getCleanValue('overlap'),
+                ]);
+
+                $sExt = pathinfo($sFileName, PATHINFO_EXTENSION);
+                $sMethod = isset($this->_aDataExt[$sExt]) ? $this->_aDataExt[$sExt] : 'getFileContentPlain';
+                $sContent = $this->$sMethod($sFileTmpName);
+
+                if (empty($sContent)) {
+                    continue;
+                }   
+                $bRet = $this->_oDb->addVectorStoreData ($iVectorStoreId, 'custom', $sFileName, (int)$sFileSize, $sMetadata, $sSettings, $sContent);
+                if ($bRet) {
+                    $iFilesAdded++;
+                }
+            }
+            return echoJson(['msg' => _t('_sys_agents_vector_store_data_queued', $iFilesAdded) . ' / num:' . $iFilesAdded]);
+
+            if($oForm->add($iId) === false)
+                return echoJson(['msg' => _t('_sys_txt_error_occured')]);
+
+            return echoJson(['grid' => $this->getCode(false), 'blink' => $iId]);
+        } 
+
+        $sFormId = $oForm->getId();
+        $sForm = $oForm->getCode(true);
+        $sContent = BxTemplStudioFunctions::getInstance()->popupBox($sFormId . '_popup_' . $sAction, _t('_sys_agents_vector_store_popup_add_data'), $this->_oTemplate->parseHtmlByName('agents_automator_form.html', [
+            'form_id' => $sFormId,
+            'form' => $sForm,
+            'object' => $this->_sObject,
+            'action' => $sAction
+        ]));
+
+        return echoJson(['popup' => ['html' => $sContent, 'options' => ['closeOnOuterClick' => false]]]);
+    }
 
     public function performActionEdit()
     {
@@ -57,6 +120,23 @@ class BxBaseStudioAgentsVectorStore extends BxDolStudioAgentsVectorStore
         return echoJson(['popup' => ['html' => $sContent, 'options' => ['closeOnOuterClick' => false]]]);
     }
 
+    public function performActionFiles()
+    {
+        $iId = $this->_getId();
+        $aVectorStore = $this->_oDb->getVectorStoreById($iId);
+
+        $oGrid = BxDolGrid::getObjectInstance('sys_studio_agents_vector_store_data');
+        $oGrid->addMarkers(['vector_store_id' => $iId]);
+        $oGrid->setBrowseParams(['vector_store_id' => $iId]);
+        $sGrid = $oGrid->getCode();
+
+        $sContent = BxTemplStudioFunctions::getInstance()->popupBox('popup_files_' . $iId, _t('_sys_agents_vector_store_popup_files'), $this->_oTemplate->parseHtmlByName('agents_popup_grid.html', [
+            'grid' => $sGrid,
+        ]));
+
+        return echoJson(['popup' => ['html' => $sContent, 'options' => ['closeOnOuterClick' => false]]]);
+    }
+
     protected function _delete ($mixedId)
     {
         $mixedResult = parent::_delete($mixedId);
@@ -110,7 +190,7 @@ class BxBaseStudioAgentsVectorStore extends BxDolStudioAgentsVectorStore
                     'checker' => [
                         'func' => 'Json',
                         'params' => ['allow_empty' => true],
-                        'error' => _t('_sys_agents_vector_store_field_params_err'),
+                        'error' => _t('_sys_agents_vector_store_json_field_err'),
                     ],
                     'db' => [
                         'pass' => 'All'
@@ -121,6 +201,95 @@ class BxBaseStudioAgentsVectorStore extends BxDolStudioAgentsVectorStore
         ];
     }
 
+    protected function _getFormAddData(string $sAction, array $aVectorStore)
+    {
+        $sJsObject = $this->getPageJsObject();
+        $aExt = array_keys($this->_aDataExt);
+        $sExtList = '.' . implode(', .', $aExt);
+
+        return [
+            'form_attrs' => [
+                'id' => 'bx_std_agents_helpers_' . $sAction,
+                'action' => BX_DOL_URL_ROOT . 'grid.php?o=sys_studio_agents_vector_store&a=' . $sAction,
+                'method' => 'post',
+            ],
+            'params' => array (
+                'db' => array(
+                    'table' => 'sys_agents_vector_store_data',
+                    'key' => 'id',
+                    'submit_name' => 'do_submit',
+                ),
+            ),
+            'inputs' => [
+                'vector_store_id' => [
+                    'type' => 'hidden',
+                    'name' => 'vector_store_id',
+                    'value' => $aVectorStore ? $aVectorStore['id'] : 0,
+                ],
+                'chunk_size' => [
+                    'type' => 'text',
+                    'name' => 'chunk_size',
+                    'caption' => '_sys_agents_vector_store_field_chunk_size',
+                    'info' => '_sys_agents_vector_store_field_chunk_size_info',
+                    'required' => true,
+                    'value' => 512,
+                    'db' => [
+                        'pass' => 'Int'
+                    ]
+                ],
+                'delimeter' => [
+                    'type' => 'text',
+                    'name' => 'delimeter',
+                    'caption' => '_sys_agents_vector_store_field_delimeter',
+                    'info' => '_sys_agents_vector_store_field_chunk_size_info',
+                    'required' => true,
+                    'value' => '.',
+                    'db' => [
+                        'pass' => 'Xss'
+                    ]
+                ],
+                'overlap' => [
+                    'type' => 'text',
+                    'name' => 'overlap',
+                    'caption' => '_sys_agents_vector_store_field_overlap',
+                    'info' => '_sys_agents_vector_store_field_chunk_size_info',
+                    'required' => true,
+                    'value' => 2,
+                    'db' => [
+                        'pass' => 'Int'
+                    ]
+                ],
+                'metadata' => [
+                    'type' => 'textarea',
+                    'name' => 'metadata',
+                    'caption' => '_sys_agents_vector_store_field_metadata',
+                    'info' => '_sys_agents_vector_store_field_chunk_size_info',
+                    'required' => false,
+                    'checker' => [
+                        'func' => 'Json',
+                        'params' => ['allow_empty' => true],
+                        'error' => _t('_sys_agents_vector_store_json_field_err'),
+                    ],
+                    'db' => [
+                        'pass' => 'All'
+                    ]
+                ],
+                'files' => [
+                    'attrs' => [
+                        'multiple' => true,
+                        'accept' => $sExtList
+                    ],
+                    'type' => 'file',
+                    'name' => 'files[]',
+                    'caption' => _t('_sys_agents_vector_store_field_files'),
+                    'caption_info' => _t('_sys_agents_vector_store_field_files_info'),
+                    'required' => true,
+                ],
+                'submit' => $this->_getFormControls(),
+            ]
+        ];
+    }
+
     protected function _getFormEdit($sAction, $aVectorStore = [])
     {
         $aForm = $this->_getForm($sAction, $aVectorStore);
@@ -129,7 +298,16 @@ class BxBaseStudioAgentsVectorStore extends BxDolStudioAgentsVectorStore
         return $aForm;
     }
 
-
+    protected function getFileContentPlain($sFilePath)
+    {
+        return file_get_contents($sFilePath);
+    }
+    protected function getFileContentHtml($sFilePath)  
+    {
+        $s = file_get_contents($sFilePath);
+        $html = new \Html2Text\Html2Text($s);
+        return $html->getText();
+    }
 }
 
 /** @} */

studio/template/scripts/BxBaseStudioAgentsVectorStoreData.php

@@ -0,0 +1,45 @@
+<?php defined('BX_DOL') or die('hack attempt');
+/**
+ * Copyright (c) UNA, Inc - https://una.io
+ * MIT License - https://opensource.org/licenses/MIT
+ *
+ * @defgroup    UnaView UNA Studio Representation classes
+ * @ingroup     UnaStudio
+ * @{
+ */
+
+class BxBaseStudioAgentsVectorStoreData extends BxDolStudioAgentsVectorStoreData
+{
+    public function __construct ($aOptions, $oTemplate = false)
+    {
+        parent::__construct ($aOptions, $oTemplate);
+        $this->addMarkers($this->_aBrowseParams);
+    }
+
+    protected function _delete ($mixedId)
+    {
+        // TODO: delete files from vector store
+
+        $mixedResult = parent::_delete($mixedId);
+        return $mixedResult;
+    }
+
+    protected function _getCellSize ($mixedValue, $sKey, $aField, $aRow) 
+    {
+        $mixedValue = bx_format_bytes($mixedValue);
+        return parent::_getCellDefault ($mixedValue, $sKey, $aField, $aRow);
+    } 
+
+    protected function _getCellStatus ($mixedValue, $sKey, $aField, $aRow) 
+    {
+        $mixedValue = $mixedValue == 'processing' ? '<span title="Processing">🕚</span>' : '<span title="Completed">✅</span>';
+        return parent::_getCellDefault ($mixedValue, $sKey, $aField, $aRow);
+    } 
+
+    static public function processPendingData()
+    {
+        // TODO: process pending data, e.g. add to vector store, update status, etc.
+    }
+}
+
+/** @} */

studio/template/scripts_templ/BxTemplStudioAgentsVectorStoreData.php

@@ -0,0 +1,19 @@
+<?php defined('BX_DOL') or die('hack attempt');
+/**
+ * Copyright (c) UNA, Inc - https://una.io
+ * MIT License - https://opensource.org/licenses/MIT
+ *
+ * @defgroup    UnaStudioView UNA Studio Representation classes
+ * @ingroup     UnaStudio
+ * @{
+ */
+
+class BxTemplStudioAgentsVectorStoreData extends BxBaseStudioAgentsVectorStoreData
+{
+    public function __construct ($aOptions, $oTemplate = false)
+    {
+        parent::__construct ($aOptions, $oTemplate);
+    }
+}
+
+/** @} */