#3068 Files: Bulk downloading reworked. Added support for delayed downloads in case of remote storage is used.

modules/boonex/files/classes/BxFilesBulkDownloader.php

@@ -0,0 +1,164 @@
+<?php defined('BX_DOL') or die('hack attempt');
+/**
+ * Copyright (c) UNA, Inc - https://una.io
+ * MIT License - https://opensource.org/licenses/MIT
+ *
+ * @defgroup    Files Files
+ * @ingroup     UnaModules
+ *
+ * @{
+ */
+
+/**
+ * Files bulk downloader
+ */
+
+define('BX_FILES_DOWNLOADER_TIMEOUT', 10); //seconds
+define('BX_FILES_DOWNLOADER_TIMEOUT_REQUESTS', 15); //seconds
+
+define('BX_FILES_DOWNLOADER_STATUS_EMPTY', 0);
+define('BX_FILES_DOWNLOADER_STATUS_TOO_LARGE', 1);
+define('BX_FILES_DOWNLOADER_STATUS_DOWNLOADING', 2);
+define('BX_FILES_DOWNLOADER_STATUS_READY', 3);
+
+class BxFilesBulkDownloader
+{
+    var $oModule;
+    public function __construct(&$oModule)
+    {
+        $this->oModule = &$oModule;
+    }
+
+    public function createDownloadingJob($aFiles, $sZipFileName) {
+        $CNF = &$this->oModule->_oConfig->CNF;
+
+        $aFilesList = $this->getFolderFiles($aFiles, '/', 'mixed');
+        if(!$aFilesList) return [
+            'status' => BX_FILES_DOWNLOADER_STATUS_EMPTY,
+        ];
+
+        $iMaxSize = intval(getParam($CNF['PARAM_MAX_BULK_DOWNLOAD_SIZE'])) * 1024 * 1024;
+        if ($iMaxSize) {
+            $iTotalSize = 0;
+            foreach ($aFilesList as $aFile) {
+                $iTotalSize += $aFile['size'];
+            }
+            if ($iTotalSize >= $iMaxSize) return [
+                'status' => BX_FILES_DOWNLOADER_STATUS_TOO_LARGE,
+            ];
+        }
+
+        $mRes = $this->processDownloading($aFilesList);
+        if ($mRes === BX_FILES_DOWNLOADER_STATUS_DOWNLOADING) {
+            $iDownloadingJobId = $this->oModule->_oDb->createDownloadingJob($aFilesList, $sZipFileName, bx_get_logged_profile_id());
+            return [
+                'status' => BX_FILES_DOWNLOADER_STATUS_DOWNLOADING,
+                'job' => $iDownloadingJobId,
+            ];
+        } else {
+            return [
+                'status' => BX_FILES_DOWNLOADER_STATUS_READY,
+                'file' => $mRes,
+                'filename' => $this->sanitizeFileName($sZipFileName).'.zip',
+            ];
+        }
+    }
+
+    public function processDownloading(&$aFiles) {
+        $CNF = &$this->oModule->_oConfig->CNF;
+
+        $oStorage = BxDolStorage::getObjectInstance($CNF['OBJECT_STORAGE']);
+
+        $iJobStarted = time();
+
+        foreach ($aFiles as $iIndex => $aFile) {
+            if (isset($aFile['downloaded']) && file_exists($aFile['downloaded'])) continue;
+
+            $sFileUrl = $oStorage->getFileUrlById($aFile['id']);
+            $sFilePath = BX_DIRECTORY_PATH_TMP . 'download-job-file-'.$aFile['id'];
+            if (!file_exists($sFilePath)) {
+                @file_put_contents($sFilePath, bx_file_get_contents($sFileUrl));
+            }
+
+            $aFiles[$iIndex]['downloaded'] = $sFilePath;
+
+            if (time() - $iJobStarted >= BX_FILES_DOWNLOADER_TIMEOUT) {
+                return BX_FILES_DOWNLOADER_STATUS_DOWNLOADING;
+            }
+        }
+
+        return $this->createZipArchive($aFiles);
+    }
+
+    private function createZipArchive(&$aFiles) {
+        $sZipFile = BX_DIRECTORY_PATH_TMP.'files-folder-'.time().mt_rand().'.zip';
+        if (file_exists($sZipFile)) @unlink($sZipFile);
+
+        $oZipFile = new ZipArchive();
+        if ($oZipFile->open($sZipFile, ZIPARCHIVE::CREATE | ZIPARCHIVE::OVERWRITE)) {
+
+            $iIndex = 0;
+            foreach ($aFiles as $aFile) {
+                $sFilePath = $aFile['downloaded'];
+                if (file_exists($sFilePath) && is_readable($sFilePath)) {
+                    $oZipFile->addFile($sFilePath, $aFile['filename']);
+                    $oZipFile->setCompressionIndex($iIndex++, ZipArchive::CM_STORE);
+                }
+            }
+            $oZipFile->close();
+
+            foreach ($aFiles as $aFile)
+                @unlink($aFile['downloaded']);
+
+            return $sZipFile;
+        } else {
+            return BX_FILES_DOWNLOADER_STATUS_EMPTY;
+        }
+    }
+
+    private function getFolderFiles($mFile, $sRelPath, $sType) {
+        $aFiles = $this->oModule->_oDb->getFolderFilesEx($mFile, $sType);
+        if (!$aFiles) return [];
+
+        $aResult = [];
+        $aUniqueNames = [];
+        foreach ($aFiles as $aFile) {
+            if (CHECK_ACTION_RESULT_ALLOWED !== $this->oModule->checkAllowedView($aFile)) continue;
+
+            //in case title has been changed we might have to add extensions manually
+            $aFileInfo = pathinfo($aFile['title']);
+            $sFileBaseName = isset($aFileInfo['filename']) ? $aFileInfo['filename'] : '_';
+            $sFileExt = isset($aFileInfo['extension']) ? $aFileInfo['extension'] : '';
+
+            if (!$sFileExt || $sFileExt != $aFile['ext']) $sFileExt = $aFile['ext'];
+
+            //handle multiple files with the same name
+            $sFilename = $this->sanitizeFileName($sFileBaseName);
+            $iIndex = 1;
+            while (isset($aUniqueNames[$sFilename])) {
+                $sFilename = $this->sanitizeFileName($sFileBaseName.'_'.$iIndex++);
+            }
+            $aUniqueNames[$sFilename] = 1;
+
+            if ($aFile['type'] == 'folder') {
+                $aResult = array_merge($aResult, $this->getFolderFiles($aFile['id'], $sRelPath . $sFilename . '/', 'folder'));
+            } else {
+                $aResult[] = [
+                    'filename' => trim($sRelPath.$sFilename.'.'.$sFileExt, '/'),
+                    'size' => $aFile['size'],
+                    'id' => $aFile['file_id'],
+                ];
+            }
+        }
+
+        return $aResult;
+    }
+
+    private function sanitizeFileName($sDesiredFilename) {
+        $sDesiredFilename = mb_ereg_replace("([^\w\s\d\-_~,;\[\]\(\).])", '_', $sDesiredFilename);
+        $sDesiredFilename = mb_ereg_replace("([\.]{2,})", '_', $sDesiredFilename);
+        return $sDesiredFilename;
+    }
+}
+
+/** @} */

modules/boonex/files/classes/BxFilesConfig.php

@@ -84,6 +84,7 @@ class BxFilesConfig extends BxBaseModTextConfig
             'PARAM_SEARCHABLE_FIELDS' => 'bx_files_searchable_fields',
             'PARAM_LINK_TO_PREVIEW' => 'bx_files_show_link_to_preview',
             'PARAM_MAX_NESTING_LEVEL' => 'bx_files_max_nesting_level',
+            'PARAM_MAX_BULK_DOWNLOAD_SIZE' => 'bx_files_max_bulk_download_size',
 
             // objects
             'OBJECT_STORAGE' => 'bx_files_files',

modules/boonex/files/classes/BxFilesCronProcessData.php

@@ -14,6 +14,7 @@ class BxFilesCronProcessData extends BxDolCron
     function processing()
     {
         BxDolService::call('bx_files', 'process_files_data', array(3));
+        BxDolService::call('bx_files', 'prune_downloading_jobs', array());
     }
 }
 

modules/boonex/files/classes/BxFilesDb.php

@@ -113,7 +113,7 @@ class BxFilesDb extends BxBaseModFilesDb
         $CNF = &$this->_oConfig->CNF;
 
         $sQuery = "
-            SELECT `{$CNF['TABLE_ENTRIES']}`.`{$CNF['FIELD_ID']}`, `{$CNF['TABLE_ENTRIES']}`.`{$CNF['FIELD_AUTHOR']}`, `{$CNF['TABLE_ENTRIES']}`.`{$CNF['FIELD_TITLE']}`, `{$CNF['TABLE_ENTRIES']}`.`type`, `{$CNF['TABLE_FILES']}`.`path`, `{$CNF['TABLE_FILES']}`.`ext`, `{$CNF['TABLE_FILES']}`.`size` 
+            SELECT `{$CNF['TABLE_ENTRIES']}`.`{$CNF['FIELD_ID']}`, `{$CNF['TABLE_ENTRIES']}`.`{$CNF['FIELD_FILE_ID']}`, `{$CNF['TABLE_ENTRIES']}`.`{$CNF['FIELD_AUTHOR']}`, `{$CNF['TABLE_ENTRIES']}`.`{$CNF['FIELD_TITLE']}`, `{$CNF['TABLE_ENTRIES']}`.`type`, `{$CNF['TABLE_FILES']}`.`path`, `{$CNF['TABLE_FILES']}`.`ext`, `{$CNF['TABLE_FILES']}`.`size` 
             FROM `{$CNF['TABLE_ENTRIES']}` 
             LEFT JOIN `{$CNF['TABLE_FILES']}` ON `{$CNF['TABLE_ENTRIES']}`.`{$CNF['FIELD_FILE_ID']}` = `{$CNF['TABLE_FILES']}`.`{$CNF['FIELD_ID']}`
         ";
@@ -197,6 +197,37 @@ class BxFilesDb extends BxBaseModFilesDb
 
         return $aResult;
     }
+
+    public function createDownloadingJob($aFiles, $sZipFileName, $iOwner) {
+        $this->query("INSERT INTO `bx_files_downloading_jobs` (`name`, `owner`, `files`, `started`) VALUES (:name, :owner, :files, UNIX_TIMESTAMP())", [
+            'files' => serialize($aFiles),
+            'owner' => $iOwner,
+            'name' => $sZipFileName
+        ]);
+        return $this->lastId();
+    }
+
+    public function getDownloadingJob($iJob) {
+        $aJob = $this->getRow("SELECT * FROM `bx_files_downloading_jobs` WHERE `id` = :id", ['id' => $iJob]);
+        if ($aJob && $aJob['files']) $aJob['files'] = unserialize($aJob['files']);
+        return $aJob;
+    }
+
+    public function updateDownloadingJob($iJob, $aFiles, $sZipFilePath) {
+        $this->query("UPDATE `bx_files_downloading_jobs` SET `result` = :result, `files` = :files WHERE `id` = :id", [
+            'result' => $sZipFilePath,
+            'files' => serialize($aFiles),
+            'id' => $iJob
+        ]);
+    }
+
+    public function deleteOldDownloadingJobs() {
+        $aFiles = $this->getColumn("SELECT `result` FROM `bx_files_downloading_jobs` WHERE UNIX_TIMESTAMP() - `started` > 24*3600");
+
+        if ($aFiles)
+            $this->query("DELETE FROM `bx_files_downloading_jobs` WHERE UNIX_TIMESTAMP() - `started` > 24*3600");
+        return $aFiles;
+    }
 }
 
 /** @} */

modules/boonex/files/classes/BxFilesModule.php

@@ -80,20 +80,30 @@ class BxFilesModule extends BxBaseModTextModule
             return;
         }
 
-        if ($aData['type'] == 'folder' || $aFilesToDownload) {
-            $sZipFilePath = $this->prepareFolderForDownloading($aFilesToDownload);
-            if ($sZipFilePath == -1) {
-                $this->_oTemplate->displayMsg(_t('_bx_files_txt_max_size_reached', _t_format_size(intval(getParam('bx_files_max_bulk_download_size')) * 1024 * 1024)));
-                return;
+        if ($aData['type'] == 'folder' || $aFilesToDownload && count($aFilesToDownload) > 1) {
+            bx_import('BulkDownloader', $this->_aModule);
+            $oDownloader = new BxFilesBulkDownloader($this);
+            $aResult = $oDownloader->createDownloadingJob($aFilesToDownload, $aData[$CNF['FIELD_TITLE']]);
+
+            switch ($aResult['status']) {
+                case BX_FILES_DOWNLOADER_STATUS_TOO_LARGE:
+                    $this->_oTemplate->displayMsg(_t('_bx_files_txt_max_size_reached', _t_format_size(intval(getParam('bx_files_max_bulk_download_size')) * 1024 * 1024)));
+                    break;
+                case BX_FILES_DOWNLOADER_STATUS_DOWNLOADING:
+                    $this->_oTemplate->addJsTranslation('_bx_files_txt_status_downloading_ready_msg');
+                    $this->_oTemplate->displayMsg(['title' => _t('_bx_files_txt_status_downloading'), 'content' => $this->_oTemplate->parseHtmlByName('bulk_downloader_process.html', [
+                        'timeout' => BX_FILES_DOWNLOADER_TIMEOUT_REQUESTS,
+                        'action_url' => BX_DOL_URL_ROOT.$this->_oConfig->getBaseUri().'process_download/'.$aResult['job'],
+                    ])]);
+                    break;
+                case BX_FILES_DOWNLOADER_STATUS_READY:
+                    bx_smart_readfile($aResult['file'], $aResult['filename'], 'application/zip');
+                    break;
+                case BX_FILES_DOWNLOADER_STATUS_EMPTY:
+                default:
+                    $this->_oTemplate->displayPageNotFound();
+                    break;
             }
-
-            if (!$sZipFilePath) {
-                $this->_oTemplate->displayPageNotFound();
-                return;
-            }
-
-            $sFolderName = $this->sanitizeFileName($aData[$CNF['FIELD_TITLE']]).'.zip';
-            bx_smart_readfile($sZipFilePath, $sFolderName, 'application/zip');
             exit;
         }
 
@@ -116,6 +126,34 @@ class BxFilesModule extends BxBaseModTextModule
         
         exit;   
     }
+
+    public function actionProcessDownload($iDownloadingJobId) {
+        if (!$iDownloadingJobId) die;
+
+        $aJob = $this->_oDb->getDownloadingJob($iDownloadingJobId);
+        if (!$aJob || $aJob['owner'] != bx_get_logged_profile_id()) die;
+
+        if ($aJob['result']) {
+            bx_smart_readfile($aJob['result'], $aJob['name'], 'application/zip');
+            exit;
+        }
+
+        bx_import('BulkDownloader', $this->_aModule);
+        $oDownloader = new BxFilesBulkDownloader($this);
+
+        $mRes = $oDownloader->processDownloading($aJob['files']);
+        if ($mRes === BX_FILES_DOWNLOADER_STATUS_DOWNLOADING) {
+            $this->_oDb->updateDownloadingJob($iDownloadingJobId, $aJob['files'], '');
+            echoJson([
+                'status' => 'downloading',
+            ]);
+        } else {
+            $this->_oDb->updateDownloadingJob($iDownloadingJobId, $aJob['files'], $mRes);
+            echoJson([
+                'status' => 'ready',
+            ]);
+        }
+    }
     
     public function getContentFile($aData) 
     {
@@ -189,6 +227,12 @@ class BxFilesModule extends BxBaseModTextModule
         }
     }
 
+    public function servicePruneDownloadingJobs()
+    {
+        $aFiles = $this->_oDb->deleteOldDownloadingJobs();
+        if ($aFiles) foreach ($aFiles as $sFile) @unlink($sFile);
+    }
+
     protected function _getContentForTimelinePost($aEvent, $aContentInfo, $aBrowseParams = array())
     {
         $aResult = parent::_getContentForTimelinePost($aEvent, $aContentInfo, $aBrowseParams);
@@ -529,87 +573,6 @@ class BxFilesModule extends BxBaseModTextModule
         }
         parent::serviceDeleteEntity($iContentId, $sFuncDelete);
     }
-
-    private function prepareFolderForDownloading($aFiles) {
-        $CNF = &$this->_oConfig->CNF;
-
-        $aFilesList = $this->getFolderFiles($aFiles, '/', 'mixed');
-        if(!$aFilesList) return false;
-
-        $iMaxSize = intval(getParam('bx_files_max_bulk_download_size')) * 1024 * 1024;
-        if ($iMaxSize) {
-            $iTotalSize = 0;
-            foreach ($aFilesList as $aFile) {
-                $iTotalSize += $aFile['size'];
-            }
-            if ($iTotalSize >= $iMaxSize) return -1;
-        }
-
-        $sZipFile = BX_DIRECTORY_PATH_TMP.'files-folder-'.mt_rand().'.zip';
-        if (file_exists($sZipFile)) @unlink($sZipFile);
-
-        $oZipFile = new ZipArchive();
-        if ($oZipFile->open($sZipFile, ZIPARCHIVE::CREATE | ZIPARCHIVE::OVERWRITE)) {
-
-            $iIndex = 0;
-            foreach ($aFilesList as $aFile) {
-                $sFilePath = BX_DIRECTORY_STORAGE . $CNF['OBJECT_STORAGE'].'/'.$aFile['path'];
-                if (file_exists($sFilePath) && is_readable($sFilePath)) {
-                    $oZipFile->addFile($sFilePath, $aFile['filename']);
-                    $oZipFile->setCompressionIndex($iIndex++, ZipArchive::CM_STORE);
-                }
-            }
-
-            $oZipFile->close();
-            return $sZipFile;
-        } else {
-            return false;
-        }
-    }
-
-    private function getFolderFiles($mFile, $sRelPath, $sType) {
-        $aFiles = $this->_oDb->getFolderFilesEx($mFile, $sType);
-        if (!$aFiles) return [];
-
-        $aResult = [];
-        $aUniqueNames = [];
-        foreach ($aFiles as $aFile) {
-            if (CHECK_ACTION_RESULT_ALLOWED !== $this->checkAllowedView($aFile)) continue;
-
-            //in case title has been changed we might have to add extensions manually
-            $aFileInfo = pathinfo($aFile['title']);
-            $sFileBaseName = isset($aFileInfo['filename']) ? $aFileInfo['filename'] : '_';
-            $sFileExt = isset($aFileInfo['extension']) ? $aFileInfo['extension'] : '';
-
-            if (!$sFileExt || $sFileExt != $aFile['ext']) $sFileExt = $aFile['ext'];
-
-            //handle multiple files with the same name
-            $sFilename = $this->sanitizeFileName($sFileBaseName);
-            $iIndex = 1;
-            while (isset($aUniqueNames[$sFilename])) {
-                $sFilename = $this->sanitizeFileName($sFileBaseName.'_'.$iIndex++);
-            }
-            $aUniqueNames[$sFilename] = 1;
-
-            if ($aFile['type'] == 'folder') {
-                $aResult = array_merge($aResult, $this->getFolderFiles($aFile['id'], $sRelPath . $sFilename . '/', 'folder'));
-            } else {
-                $aResult[] = [
-                    'filename' => trim($sRelPath.$sFilename.'.'.$sFileExt, '/'),
-                    'size' => $aFile['size'],
-                    'path' => $aFile['path'],
-                ];
-            }
-        }
-
-        return $aResult;
-    }
-
-    private function sanitizeFileName($sDesiredFilename) {
-        $sDesiredFilename = mb_ereg_replace("([^\w\s\d\-_~,;\[\]\(\).])", '_', $sDesiredFilename);
-        $sDesiredFilename = mb_ereg_replace("([\.]{2,})", '_', $sDesiredFilename);
-        return $sDesiredFilename;
-    }
 }
 
 /** @} */

modules/boonex/files/install/langs/en.xml

@@ -291,4 +291,8 @@
     <string name="_bx_files_bulk_action_title_delete"><![CDATA[Delete]]></string>
     <string name="_bx_files_bulk_action_title_move_to"><![CDATA[Move To...]]></string>
 
+    <string name="_bx_files_txt_status_downloading"><![CDATA[Downloading...]]></string>
+    <string name="_bx_files_txt_status_downloading_msg"><![CDATA[The download will start shortly...]]></string>
+    <string name="_bx_files_txt_status_downloading_ready_msg"><![CDATA[Click <a href={0}>here</a> if the download doesn't start automatically.]]></string>
+
 </resources>

modules/boonex/files/install/sql/install.sql

@@ -97,6 +97,17 @@ CREATE TABLE IF NOT EXISTS `bx_files_bookmarks` (
   PRIMARY KEY (`object_id`, `profile_id`)
 );
 
+-- TABLE: downloading jobs
+CREATE TABLE IF NOT EXISTS `bx_files_downloading_jobs` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `name` varchar(255) NOT NULL,
+  `owner` int(11) NOT NULL,
+  `files` MEDIUMTEXT NOT NULL,
+  `result` TEXT NOT NULL,
+  `started` int(11) NOT NULL,
+  PRIMARY KEY (`id`)
+);
+
 -- TABLE: votes
 CREATE TABLE IF NOT EXISTS `bx_files_votes` (
   `object_id` int(11) NOT NULL default '0',

modules/boonex/files/install/sql/uninstall.sql

@@ -1,6 +1,6 @@
 
 -- TABLES
-DROP TABLE IF EXISTS `bx_files_main`, `bx_files_files`, `bx_files_photos_resized`, `bx_files_cmts`, `bx_files_votes`, `bx_files_votes_track`, `bx_files_reactions`, `bx_files_reactions_track`, `bx_files_views_track`, `bx_files_meta_keywords`, `bx_files_meta_mentions`, `bx_files_reports`, `bx_files_reports_track`, `bx_files_favorites_track`, `bx_files_favorites_lists`, `bx_files_scores`, `bx_files_scores_track`, `bx_files_bookmarks`;
+DROP TABLE IF EXISTS `bx_files_main`, `bx_files_files`, `bx_files_photos_resized`, `bx_files_cmts`, `bx_files_votes`, `bx_files_votes_track`, `bx_files_reactions`, `bx_files_reactions_track`, `bx_files_views_track`, `bx_files_meta_keywords`, `bx_files_meta_mentions`, `bx_files_reports`, `bx_files_reports_track`, `bx_files_favorites_track`, `bx_files_favorites_lists`, `bx_files_scores`, `bx_files_scores_track`, `bx_files_bookmarks`, `bx_files_downloading_jobs`;
 
 -- STORAGES & TRANSCODERS
 DELETE FROM `sys_objects_storage` WHERE `object` = 'bx_files_files' OR `object` = 'bx_files_photos_resized';

modules/boonex/files/template/bulk_downloader_process.html

@@ -0,0 +1,13 @@
+<span id="bx-files-downloading-message"><bx_text:_bx_files_txt_status_downloading_msg /></span>
+<script>
+    var bx_files_h_interval = 0;
+    bx_files_h_interval = setInterval(function(){
+        $.get('__action_url__', function(oResult) {
+            if (oResult.status == 'ready') {
+                clearInterval(bx_files_h_interval);
+                window.location.href = '__action_url__';
+                $('#bx-files-downloading-message').html(_t('_bx_files_txt_status_downloading_ready_msg', '__action_url__'));
+            }
+        }, 'json');
+    }, __timeout__*1000);
+</script>

modules/boonex/russian/data/langs/bx_files/ru.xml

@@ -287,4 +287,8 @@
     <string name="_bx_files_bulk_action_title_bookmark"><![CDATA[Сделать закладки]]></string>
     <string name="_bx_files_bulk_action_title_delete"><![CDATA[Удалить]]></string>
     <string name="_bx_files_bulk_action_title_move_to"><![CDATA[Переместить в...]]></string>
+
+    <string name="_bx_files_txt_status_downloading"><![CDATA[Скачивание...]]></string>
+    <string name="_bx_files_txt_status_downloading_msg"><![CDATA[Скачивание скоро начнется...]]></string>
+    <string name="_bx_files_txt_status_downloading_ready_msg"><![CDATA[Нажмите <a href={0}>здесь</a>, если скачивание не началось автоматически.]]></string>
 </resources>