BxAnalyticsDb.php 130 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722
  1. <?php defined('BX_DOL') or die('hack attempt');
  2. /**
  3. * Copyright (c) UNA, Inc - https://una.io
  4. * MIT License - https://opensource.org/licenses/MIT
  5. *
  6. * @defgroup Analytics Analytics
  7. * @ingroup UnaModules
  8. *
  9. * @{
  10. */
  11. /*
  12. * Module database queries
  13. */
  14. class BxAnalyticsDb extends BxBaseModGeneralDb
  15. {
  16. /**
  17. * Report dimensions stored on sessions, as name => column. Filters and top lists accept
  18. * these names only, so no user input reaches SQL as an identifier.
  19. */
  20. protected $_aSessionDimensions = [
  21. 'platform' => 'platform',
  22. 'app' => 'app',
  23. 'app_version' => 'app_version',
  24. 'hostname' => 'hostname',
  25. 'entry_path' => 'entry_path',
  26. 'exit_path' => 'exit_path',
  27. 'referrer' => 'referrer',
  28. 'referrer_host' => 'referrer_host',
  29. 'channel' => 'channel',
  30. 'utm_source' => 'utm_source',
  31. 'utm_medium' => 'utm_medium',
  32. 'utm_campaign' => 'utm_campaign',
  33. 'utm_term' => 'utm_term',
  34. 'utm_content' => 'utm_content',
  35. 'country' => 'country',
  36. 'region' => 'region',
  37. 'city' => 'city',
  38. 'language' => 'language',
  39. 'browser' => 'browser',
  40. 'os' => 'os',
  41. 'device' => 'device',
  42. 'screen' => 'screen',
  43. ];
  44. /**
  45. * Report dimensions stored on pageview events.
  46. */
  47. protected $_aEventDimensions = [
  48. 'path' => 'path',
  49. 'title' => 'title',
  50. ];
  51. /**
  52. * What a breakdown of a session dimension may be sorted by, as metric => SQL over the
  53. * sessions (alias `s`). Fixed expressions: the metric name from the request only picks one.
  54. */
  55. protected $_aSessionMetrics = [
  56. 'visitors' => 'COUNT(DISTINCT `s`.`visitor`)',
  57. 'sessions' => 'COUNT(*)',
  58. 'pageviews' => 'SUM(`s`.`pageviews`)',
  59. 'pages_per_session' => 'SUM(`s`.`pageviews`) / COUNT(*)',
  60. 'bounce_rate' => 'SUM(`s`.`bounce`) / COUNT(*)',
  61. 'duration' => 'SUM(`s`.`duration`) / COUNT(*)',
  62. ];
  63. /**
  64. * The same for an event dimension, over pageview events (alias `e`).
  65. */
  66. protected $_aEventMetrics = [
  67. 'visitors' => 'COUNT(DISTINCT `e`.`visitor`)',
  68. 'pageviews' => 'COUNT(*)',
  69. 'engaged' => 'IFNULL(AVG(NULLIF(`e`.`engaged`, 0)), 0)',
  70. ];
  71. /**
  72. * Pages and titles grouped with the number of their visitors: sorted first, then counted.
  73. * Read in time order (the `type_ts` key, which the optimizer takes for a short period or
  74. * with a visit filter) and grouped in a temporary table, a 255 character key with a
  75. * distinct count per group spilled to disk: 160 s for 30 days of 10 million events, against
  76. * 1 to 4 s sorted.
  77. */
  78. const SQL_GROUP_WIDE = 'SQL_BIG_RESULT';
  79. /**
  80. * Optimizer hint for an IN subquery: run it once into a temporary table (materialization),
  81. * not as a join nor once per row of the outer query. Next to NO_SEMIJOIN, MariaDB
  82. * ignores it as conflicting. See _getVisitsInSql.
  83. */
  84. const SQL_SUBQUERY_FIRST = '/*+ SUBQUERY(MATERIALIZATION) */';
  85. /**
  86. * Whether the report session settings were applied on this connection.
  87. */
  88. protected static $_bReportSession = false;
  89. /**
  90. * Session settings for report queries, once per connection. MariaDB, and MySQL before
  91. * 8.0.20, sort rows wider than max_length_for_sort_data (1024 bytes) by row id and read
  92. * every row again after the sort: a top list of pages (a 255 character utf8mb4 path)
  93. * over 90 days then reads each of its pageviews twice, in random order. A larger limit
  94. * keeps the columns in the sort. MySQL 8.0.20 and later always do that and deprecated
  95. * the variable, so it is left alone there.
  96. */
  97. public function setReportSession()
  98. {
  99. if(self::$_bReportSession)
  100. return;
  101. self::$_bReportSession = true;
  102. $sVersion = (string)BxDolDb::getLink()->getAttribute(PDO::ATTR_SERVER_VERSION);
  103. if(stripos($sVersion, 'mariadb') === false && !version_compare($sVersion, '8.0.20', '<'))
  104. return;
  105. $this->query("SET SESSION `max_length_for_sort_data` = 8192");
  106. }
  107. public function isDimension($sName)
  108. {
  109. return isset($this->_aSessionDimensions[$sName]) || isset($this->_aEventDimensions[$sName]);
  110. }
  111. public function isEventDimension($sName)
  112. {
  113. return isset($this->_aEventDimensions[$sName]);
  114. }
  115. /**
  116. * Every report dimension, as name => kind (event or session): pages first.
  117. */
  118. public function getDimensions()
  119. {
  120. $aResult = [];
  121. foreach(array_keys($this->_aEventDimensions) as $sName)
  122. $aResult[$sName] = 'event';
  123. foreach(array_keys($this->_aSessionDimensions) as $sName)
  124. $aResult[$sName] = 'session';
  125. return $aResult;
  126. }
  127. /**
  128. * Metrics a breakdown of a dimension may be sorted by, the default first; empty for an
  129. * unknown dimension.
  130. */
  131. public function getBreakdownMetrics($sDimension)
  132. {
  133. if(isset($this->_aEventDimensions[$sDimension]))
  134. return array_keys($this->_aEventMetrics);
  135. return isset($this->_aSessionDimensions[$sDimension]) ? array_keys($this->_aSessionMetrics) : [];
  136. }
  137. /*
  138. * Collecting
  139. */
  140. /**
  141. * The salt for a day, created by the first hit of that day.
  142. */
  143. public function getSalt($sDay)
  144. {
  145. $aSalts = $this->getSalts([$sDay]);
  146. return $aSalts[$sDay];
  147. }
  148. /**
  149. * The salts of some days, [day => salt]. Every hit needs them and only the first hit of a
  150. * day creates one, so they are read first, in one query, and written only when missing.
  151. */
  152. public function getSalts($aDays)
  153. {
  154. $sTable = $this->_oConfig->CNF['TABLE_SALTS'];
  155. $aBindings = [];
  156. $sIn = $this->_getInSql(array_values(array_unique($aDays)), 'd', $aBindings);
  157. $aSalts = $this->getPairs("SELECT `day`, `salt` FROM `" . $sTable . "` WHERE `day` IN (" . $sIn . ")", 'day', 'salt', $aBindings);
  158. $aResult = [];
  159. foreach($aDays as $sDay) {
  160. if(!isset($aSalts[$sDay])) {
  161. $this->query("INSERT IGNORE INTO `" . $sTable . "` (`day`, `salt`) VALUES (:day, :salt)", [
  162. 'day' => $sDay,
  163. 'salt' => random_bytes(32),
  164. ]);
  165. $aSalts[$sDay] = $this->getOne("SELECT `salt` FROM `" . $sTable . "` WHERE `day` = :day", ['day' => $sDay]);
  166. }
  167. $aResult[$sDay] = (string)$aSalts[$sDay];
  168. }
  169. return $aResult;
  170. }
  171. /**
  172. * Latest session of any of the visitor ids on a platform, active since a time.
  173. */
  174. public function getActiveSession($aVisitors, $sPlatform, $iSince, $iUntil = 0)
  175. {
  176. $aBindings = ['platform' => $sPlatform, 'since' => $iSince];
  177. $sUntil = '';
  178. if($iUntil) {
  179. $sUntil = ' AND `started` <= :until';
  180. $aBindings['until'] = (int)$iUntil;
  181. }
  182. $aIn = [];
  183. foreach(array_values(array_unique($aVisitors)) as $i => $sVisitor) {
  184. $aIn[] = ':v' . $i;
  185. $aBindings['v' . $i] = $sVisitor;
  186. }
  187. if(empty($aIn))
  188. return [];
  189. return $this->getRow("SELECT * FROM `" . $this->_oConfig->CNF['TABLE_SESSIONS'] . "` WHERE `visitor` IN (" . implode(', ', $aIn) . ") AND `platform` = :platform AND `last_seen` >= :since" . $sUntil . " ORDER BY `last_seen` DESC LIMIT 1", $aBindings);
  190. }
  191. public function insertSession($aSession)
  192. {
  193. unset($aSession['id']);
  194. $aBindings = [];
  195. foreach($aSession as $sField => $mixedValue)
  196. $aBindings[$sField] = $mixedValue;
  197. $sFields = '`' . implode('`, `', array_keys($aBindings)) . '`';
  198. $sValues = ':' . implode(', :', array_keys($aBindings));
  199. if(!$this->query("INSERT INTO `" . $this->_oConfig->CNF['TABLE_SESSIONS'] . "` (" . $sFields . ") VALUES (" . $sValues . ")", $aBindings))
  200. return 0;
  201. return (int)$this->lastId();
  202. }
  203. public function updateSession($iId, $aSet)
  204. {
  205. $aBindings = ['id' => (int)$iId];
  206. $aSql = [];
  207. foreach($aSet as $sField => $mixedValue) {
  208. $aSql[] = '`' . $sField . '` = :f_' . $sField;
  209. $aBindings['f_' . $sField] = $mixedValue;
  210. }
  211. return (int)$this->query("UPDATE `" . $this->_oConfig->CNF['TABLE_SESSIONS'] . "` SET " . implode(', ', $aSql) . " WHERE `id` = :id", $aBindings) > 0;
  212. }
  213. /**
  214. * Add a request's hits to a visit's counters in the database, not from a copy read before
  215. * them: two requests of one visit at once (tabs opened together, an event and a pageview
  216. * sent together) both count. The time only moves forward, and the exit page is the visit's
  217. * latest pageview by time, whichever request is written last.
  218. *
  219. * Every expression reads only columns set after it in the statement, so the result is the
  220. * same whether the server assigns left to right or all at once (MariaDB's
  221. * SIMULTANEOUS_ASSIGNMENT).
  222. *
  223. * @param int $iPageviews pageviews in the request
  224. * @param int $iEvents events in the request
  225. * @param bool $bInteractive a hit that isn't a bounce on its own (an interactive event)
  226. * @param int $iLastSeen the request's latest hit, 0 for none
  227. * @param bool $bPageview the request has a pageview, so the exit page may change
  228. */
  229. public function addSessionHits($iId, $iPageviews, $iEvents, $bInteractive, $iLastSeen, $bPageview)
  230. {
  231. $sExit = '';
  232. $aBindings = [
  233. 'id' => (int)$iId,
  234. 'interactive' => $bInteractive ? 1 : 0,
  235. 'pv' => (int)$iPageviews,
  236. 'pv2' => (int)$iPageviews,
  237. 'ev' => (int)$iEvents,
  238. 'ls' => (int)$iLastSeen,
  239. 'ls2' => (int)$iLastSeen,
  240. ];
  241. if($bPageview) {
  242. // the `session_id` key, a visit has few rows
  243. $sExit = ", `exit_path` = IFNULL((SELECT `path` FROM `" . $this->_oConfig->CNF['TABLE_EVENTS'] . "` WHERE `session_id` = :sid AND `type` = 'pageview' ORDER BY `ts` DESC, `id` DESC LIMIT 1), `exit_path`)";
  244. $aBindings['sid'] = (int)$iId;
  245. }
  246. return (int)$this->query("UPDATE `" . $this->_oConfig->CNF['TABLE_SESSIONS'] . "` SET
  247. `bounce` = IF(:interactive = 1 OR `pageviews` + :pv > 1, 0, `bounce`),
  248. `duration` = GREATEST(`duration`, CAST(GREATEST(`last_seen`, :ls) AS SIGNED) - CAST(`started` AS SIGNED)),
  249. `last_seen` = GREATEST(`last_seen`, :ls2),
  250. `pageviews` = LEAST(`pageviews` + :pv2, 65535),
  251. `events` = LEAST(`events` + :ev, 65535)" . $sExit . "
  252. WHERE `id` = :id", $aBindings) > 0;
  253. }
  254. /**
  255. * Take a named lock on the connection, waiting up to a number of seconds. MySQL releases it
  256. * when the connection closes, so a request that dies can't keep it.
  257. *
  258. * @return bool the lock is held
  259. */
  260. public function lockVisit($sName, $iTimeout)
  261. {
  262. return (int)$this->getOne("SELECT GET_LOCK(:name, :timeout)", ['name' => $sName, 'timeout' => (int)$iTimeout]) === 1;
  263. }
  264. public function unlockVisit($sName)
  265. {
  266. return (int)$this->getOne("SELECT RELEASE_LOCK(:name)", ['name' => $sName]) === 1;
  267. }
  268. /**
  269. * Give every event of a visit a visitor id and a profile: the member's, when they sign in
  270. * during a guest's visit. A visit has few events, found by the `session_id` key.
  271. */
  272. public function assignSessionEvents($iSessionId, $sVisitor, $iProfileId)
  273. {
  274. return (int)$this->query("UPDATE `" . $this->_oConfig->CNF['TABLE_EVENTS'] . "` SET `visitor` = :visitor, `profile_id` = :profile WHERE `session_id` = :id", ['visitor' => $sVisitor, 'profile' => (int)$iProfileId, 'id' => (int)$iSessionId]);
  275. }
  276. /**
  277. * Insert events in one query.
  278. *
  279. * @param array $aRows rows with the same keys
  280. * @return int rows inserted
  281. */
  282. public function insertEvents($aRows)
  283. {
  284. if(empty($aRows))
  285. return 0;
  286. $aFields = array_keys(reset($aRows));
  287. $aBindings = [];
  288. $aValues = [];
  289. foreach(array_values($aRows) as $i => $aRow) {
  290. $aPlaceholders = [];
  291. foreach($aFields as $sField) {
  292. $aPlaceholders[] = ':' . $sField . '_' . $i;
  293. $aBindings[$sField . '_' . $i] = $aRow[$sField];
  294. }
  295. $aValues[] = '(' . implode(', ', $aPlaceholders) . ')';
  296. }
  297. return (int)$this->query("INSERT INTO `" . $this->_oConfig->CNF['TABLE_EVENTS'] . "` (`" . implode('`, `', $aFields) . "`) VALUES " . implode(', ', $aValues), $aBindings);
  298. }
  299. /**
  300. * Add engaged time to the latest view of a page in a session.
  301. */
  302. public function addEngagedTime($iSessionId, $sPath, $iMs, $iMaxMs)
  303. {
  304. return (int)$this->query("UPDATE `" . $this->_oConfig->CNF['TABLE_EVENTS'] . "` SET `engaged` = LEAST(`engaged` + :ms, :max) WHERE `session_id` = :session AND `type` = 'pageview' AND `path` = :path ORDER BY `id` DESC LIMIT 1", [
  305. 'ms' => (int)$iMs,
  306. 'max' => (int)$iMaxMs,
  307. 'session' => (int)$iSessionId,
  308. 'path' => $sPath,
  309. ]) > 0;
  310. }
  311. /**
  312. * Count one bot request: a daily total per bot and platform, and per bot category and page.
  313. */
  314. public function addBotHit($sDay, $sBot, $sCategory, $sPlatform, $sPath, $iTime)
  315. {
  316. $CNF = &$this->_oConfig->CNF;
  317. $bResult = $this->query("INSERT INTO `" . $CNF['TABLE_BOTS'] . "` (`day`, `bot`, `category`, `platform`, `hits`, `last_seen`) VALUES (:day, :bot, :category, :platform, 1, :ts) ON DUPLICATE KEY UPDATE `hits` = `hits` + 1, `last_seen` = GREATEST(`last_seen`, VALUES(`last_seen`))", [
  318. 'day' => $sDay,
  319. 'bot' => $sBot,
  320. 'category' => $sCategory,
  321. 'platform' => $sPlatform,
  322. 'ts' => (int)$iTime,
  323. ]) !== false;
  324. if($sPath !== '')
  325. $this->query("INSERT INTO `" . $CNF['TABLE_BOT_PAGES'] . "` (`day`, `category`, `path_hash`, `path`, `hits`) VALUES (:day, :category, :hash, :path, 1) ON DUPLICATE KEY UPDATE `hits` = `hits` + 1", [
  326. 'day' => $sDay,
  327. 'category' => $sCategory,
  328. 'hash' => substr(hash('sha256', $sPath, true), 0, 8),
  329. 'path' => $sPath,
  330. ]);
  331. return $bResult;
  332. }
  333. /**
  334. * Replace the module's subscriptions to per-module actions (content added) with the given
  335. * unit => [actions] list. Fixed subscriptions from enable.sql with the same action name
  336. * (comment/added) are listed in `$aFixedUnits` and stay.
  337. *
  338. * @return int subscriptions written
  339. */
  340. public function setActionAlerts($sHandler, $aActions, $aUnits, $aFixedUnits = [])
  341. {
  342. $iHandlerId = (int)$this->getOne("SELECT `id` FROM `sys_alerts_handlers` WHERE `name` = :name LIMIT 1", ['name' => $sHandler]);
  343. if(!$iHandlerId)
  344. return 0;
  345. $aIn = [];
  346. $aBindings = ['handler' => $iHandlerId];
  347. foreach(array_values($aActions) as $i => $sAction) {
  348. $aIn[] = ':a' . $i;
  349. $aBindings['a' . $i] = $sAction;
  350. }
  351. $sKeep = '';
  352. foreach(array_values($aFixedUnits) as $i => $sUnit) {
  353. $sKeep .= ($sKeep === '' ? '' : ', ') . ':k' . $i;
  354. $aBindings['k' . $i] = $sUnit;
  355. }
  356. $this->query("DELETE FROM `sys_alerts` WHERE `handler_id` = :handler AND `action` IN (" . implode(', ', $aIn) . ")" . ($sKeep !== '' ? " AND `unit` NOT IN (" . $sKeep . ")" : ''), $aBindings);
  357. $iWritten = 0;
  358. foreach($aUnits as $sUnit => $aUnitActions)
  359. foreach($aUnitActions as $sAction)
  360. $iWritten += (int)$this->query("INSERT INTO `sys_alerts` (`unit`, `action`, `handler_id`) VALUES (:unit, :action, :handler)", [
  361. 'unit' => $sUnit,
  362. 'action' => $sAction,
  363. 'handler' => $iHandlerId,
  364. ]);
  365. return $iWritten;
  366. }
  367. /*
  368. * Maintenance
  369. */
  370. public function deleteBotsBefore($sDay)
  371. {
  372. $CNF = &$this->_oConfig->CNF;
  373. return (int)$this->query("DELETE FROM `" . $CNF['TABLE_BOTS'] . "` WHERE `day` < :day", ['day' => $sDay]) + (int)$this->query("DELETE FROM `" . $CNF['TABLE_BOT_PAGES'] . "` WHERE `day` < :day", ['day' => $sDay]);
  374. }
  375. /**
  376. * Add to a rate limit counter and return its new value. One upsert by the primary key;
  377. * LAST_INSERT_ID(expr) hands the value back without reading the row again.
  378. *
  379. * @param string $sKey 8 bytes, see BxAnalyticsCollector::_getLimitKey
  380. * @param int $iBucket the minute's or hour's start
  381. * @param int $iHits how much to add
  382. * @return int the counter after the addition, 0 when it couldn't be written
  383. */
  384. public function addLimitHits($sKey, $iBucket, $iHits)
  385. {
  386. $iHits = max(1, (int)$iHits);
  387. $mixedResult = $this->query("INSERT INTO `" . $this->_oConfig->CNF['TABLE_LIMITS'] . "` (`key`, `bucket`, `hits`) VALUES (:key, :bucket, LAST_INSERT_ID(:hits)) ON DUPLICATE KEY UPDATE `hits` = LAST_INSERT_ID(`hits` + :add)", [
  388. 'key' => $sKey,
  389. 'bucket' => (int)$iBucket,
  390. 'hits' => $iHits,
  391. 'add' => $iHits,
  392. ]);
  393. if($mixedResult === false)
  394. return 0;
  395. return (int)$this->getOne("SELECT LAST_INSERT_ID()");
  396. }
  397. /**
  398. * A rate limit counter's value, without adding to it.
  399. */
  400. public function getLimitHits($sKey, $iBucket)
  401. {
  402. return (int)$this->getOne("SELECT `hits` FROM `" . $this->_oConfig->CNF['TABLE_LIMITS'] . "` WHERE `key` = :key AND `bucket` = :bucket", ['key' => $sKey, 'bucket' => (int)$iBucket]);
  403. }
  404. /**
  405. * Delete up to `$iLimit` rate limit counters of minutes and hours that started before a time.
  406. */
  407. public function deleteLimitsBefore($iTime, $iLimit)
  408. {
  409. return (int)$this->query("DELETE FROM `" . $this->_oConfig->CNF['TABLE_LIMITS'] . "` WHERE `bucket` < :ts ORDER BY `bucket` LIMIT " . (int)$iLimit, ['ts' => (int)$iTime]);
  410. }
  411. public function deleteSaltsBefore($sDay)
  412. {
  413. return (int)$this->query("DELETE FROM `" . $this->_oConfig->CNF['TABLE_SALTS'] . "` WHERE `day` < :day", ['day' => $sDay]);
  414. }
  415. /**
  416. * Delete up to `$iLimit` events or sessions older than a time; returns how many went.
  417. */
  418. public function deleteEventsBefore($iTime, $iLimit)
  419. {
  420. return (int)$this->query("DELETE FROM `" . $this->_oConfig->CNF['TABLE_EVENTS'] . "` WHERE `ts` < :ts ORDER BY `ts` LIMIT " . (int)$iLimit, ['ts' => (int)$iTime]);
  421. }
  422. public function deleteSessionsBefore($iTime, $iLimit)
  423. {
  424. return (int)$this->query("DELETE FROM `" . $this->_oConfig->CNF['TABLE_SESSIONS'] . "` WHERE `last_seen` < :ts ORDER BY `last_seen` LIMIT " . (int)$iLimit, ['ts' => (int)$iTime]);
  425. }
  426. /**
  427. * Unlink a deleted profile: its visits and events stay as anonymous visitors'. Each visit
  428. * gets a visitor id of its own, so its visits aren't tied to one another, and the events
  429. * of a visit recorded before signing in lose the guest's id too (in persistent mode the
  430. * device's, which the device sends again on later visits). Site actions lose the account
  431. * or profile they were about (sign-up, sign-in, the profile created).
  432. *
  433. * @param string $sSalt random bytes the new ids are made from, thrown away after
  434. */
  435. public function forgetProfile($iProfileId, $sSalt)
  436. {
  437. $CNF = &$this->_oConfig->CNF;
  438. $aBindings = ['profile' => (int)$iProfileId, 'salt' => bin2hex($sSalt)];
  439. $sAbout = "IF(`e`.`type` = 'una' AND (`e`.`module` = 'account' OR `e`.`name` = 'profile_add'), 0, `e`.`object_id`)";
  440. // the events of its visits, through the visits; then its events outside them
  441. $iEvents = (int)$this->query("UPDATE `" . $CNF['TABLE_EVENTS'] . "` AS `e` INNER JOIN `" . $CNF['TABLE_SESSIONS'] . "` AS `s` ON `s`.`id` = `e`.`session_id` SET `e`.`object_id` = " . $sAbout . ", `e`.`profile_id` = 0, `e`.`visitor` = UNHEX(MD5(CONCAT(:salt, 's', `e`.`session_id`))) WHERE `s`.`profile_id` = :profile", $aBindings);
  442. $iEvents += (int)$this->query("UPDATE `" . $CNF['TABLE_EVENTS'] . "` AS `e` SET `e`.`object_id` = " . $sAbout . ", `e`.`profile_id` = 0, `e`.`visitor` = UNHEX(MD5(CONCAT(:salt, 'e', `e`.`id`))) WHERE `e`.`profile_id` = :profile", $aBindings);
  443. // the profile's creation recorded while visits weren't tied to members
  444. $this->query("UPDATE `" . $CNF['TABLE_EVENTS'] . "` SET `object_id` = 0 WHERE `type` = 'una' AND `name` = 'profile_add' AND `object_id` = :profile", ['profile' => (int)$iProfileId]);
  445. $iErrors = (int)$this->query("UPDATE `" . $CNF['TABLE_ERROR_HITS'] . "` SET `profile_id` = 0, `visitor` = IF(`session_id` > 0, UNHEX(MD5(CONCAT(:salt, 's', `session_id`))), UNHEX(MD5(CONCAT(:salt_hit, 'h', `id`)))) WHERE `profile_id` = :profile", $aBindings + ['salt_hit' => $aBindings['salt']]);
  446. // the visits last: the events above were found through them
  447. $iSessions = (int)$this->query("UPDATE `" . $CNF['TABLE_SESSIONS'] . "` SET `profile_id` = 0, `visitor` = UNHEX(MD5(CONCAT(:salt, 's', `id`))) WHERE `profile_id` = :profile", $aBindings);
  448. return $iSessions + $iEvents + $iErrors;
  449. }
  450. /**
  451. * Delete up to `$iLimit` error hits or vitals older than a time; returns how many went.
  452. */
  453. public function deleteErrorHitsBefore($iTime, $iLimit)
  454. {
  455. return (int)$this->query("DELETE FROM `" . $this->_oConfig->CNF['TABLE_ERROR_HITS'] . "` WHERE `ts` < :ts ORDER BY `ts` LIMIT " . (int)$iLimit, ['ts' => (int)$iTime]);
  456. }
  457. public function deleteVitalsBefore($iTime, $iLimit)
  458. {
  459. return (int)$this->query("DELETE FROM `" . $this->_oConfig->CNF['TABLE_VITALS'] . "` WHERE `ts` < :ts ORDER BY `ts` LIMIT " . (int)$iLimit, ['ts' => (int)$iTime]);
  460. }
  461. /**
  462. * Error groups last seen before a time: their hits are gone too.
  463. */
  464. public function deleteErrorGroupsBefore($iTime, $iLimit)
  465. {
  466. return (int)$this->query("DELETE FROM `" . $this->_oConfig->CNF['TABLE_ERRORS'] . "` WHERE `last_seen` < :ts ORDER BY `last_seen` LIMIT " . (int)$iLimit, ['ts' => (int)$iTime]);
  467. }
  468. /**
  469. * Drop the sample stacks of error groups taken before a time: a group that keeps
  470. * happening stays, but its stack, which may hold what a visitor typed, is kept no longer
  471. * than the rest of the data. The next occurrence with a stack takes a new one.
  472. */
  473. public function clearErrorStacksBefore($iTime, $iLimit)
  474. {
  475. return (int)$this->query("UPDATE `" . $this->_oConfig->CNF['TABLE_ERRORS'] . "` SET `stack` = '', `stack_ts` = 0 WHERE `stack_ts` > 0 AND `stack_ts` < :ts LIMIT " . (int)$iLimit, ['ts' => (int)$iTime]);
  476. }
  477. /*
  478. * Reports. A filter is an array from BxAnalyticsFilter::get(): from, to (unix times,
  479. * `to` exclusive), platform ('' for all) and dims (dimension => value).
  480. */
  481. /**
  482. * Visitors, sessions, pageviews, bounces and total duration in a period.
  483. */
  484. public function getTotals($aFilter)
  485. {
  486. $aBindings = [];
  487. $sWhere = $this->_getSessionsWhere($aFilter, $aBindings);
  488. $aRow = $this->getRow("SELECT COUNT(DISTINCT `s`.`visitor`) AS `visitors`, COUNT(*) AS `sessions`, IFNULL(SUM(`s`.`pageviews`), 0) AS `pageviews`, IFNULL(SUM(`s`.`bounce`), 0) AS `bounces`, IFNULL(SUM(`s`.`duration`), 0) AS `duration` FROM " . $this->_getSessionsFrom() . " WHERE " . $sWhere, $aBindings);
  489. // filtered by a page: count that page's views, not all views of the sessions that saw it
  490. if(!empty($aRow) && $this->_hasEventDimensions($aFilter))
  491. $aRow['pageviews'] = $this->_getPageviews($aFilter);
  492. return $aRow;
  493. }
  494. /**
  495. * The same metrics per time bucket.
  496. *
  497. * @param array $aFilter report filter
  498. * @param string $sBucket hour, day, week or month
  499. * @param int $iOffset timezone offset in seconds, for bucket boundaries
  500. * @param array $aStarts exact bucket starts (unix times, ascending), required for months
  501. * and used for any bucket when given
  502. * @return array rows keyed by bucket start time
  503. */
  504. public function getSeries($aFilter, $sBucket, $iOffset, $aStarts = [])
  505. {
  506. $aBindings = [];
  507. $sWhere = $this->_getSessionsWhere($aFilter, $aBindings);
  508. $sBucketSql = $this->_getBucketSql('`s`.`started`', $sBucket, $iOffset, $aStarts, $aBindings);
  509. $aRows = $this->getAllWithKey("SELECT " . $sBucketSql . " AS `bucket`, COUNT(DISTINCT `s`.`visitor`) AS `visitors`, COUNT(*) AS `sessions`, IFNULL(SUM(`s`.`pageviews`), 0) AS `pageviews`, IFNULL(SUM(`s`.`bounce`), 0) AS `bounces`, IFNULL(SUM(`s`.`duration`), 0) AS `duration` FROM " . $this->_getSessionsFrom() . " WHERE " . $sWhere . " GROUP BY `bucket` ORDER BY `bucket`", 'bucket', $aBindings);
  510. if(!$this->_hasEventDimensions($aFilter))
  511. return $aRows;
  512. // filtered by a page: that page's views in each bucket, as getTotals counts them, not
  513. // all views of the visits that saw it; a bucket may have views and no visit started in it
  514. $aEventBindings = [];
  515. $sEventWhere = $this->_getEventsWhere($aFilter, $aEventBindings);
  516. $sEventBucketSql = $this->_getBucketSql('`e`.`ts`', $sBucket, $iOffset, $aStarts, $aEventBindings);
  517. $aPageviews = $this->getPairs("SELECT " . $sEventBucketSql . " AS `bucket`, COUNT(*) AS `pageviews` FROM `" . $this->_oConfig->CNF['TABLE_EVENTS'] . "` AS `e` WHERE " . $sEventWhere . " GROUP BY `bucket`", 'bucket', 'pageviews', $aEventBindings);
  518. foreach($aRows as $mixedBucket => $aRow)
  519. $aRows[$mixedBucket]['pageviews'] = isset($aPageviews[$mixedBucket]) ? (int)$aPageviews[$mixedBucket] : 0;
  520. foreach($aPageviews as $mixedBucket => $iPageviews)
  521. if(!isset($aRows[$mixedBucket]))
  522. $aRows[$mixedBucket] = ['bucket' => $mixedBucket, 'visitors' => 0, 'sessions' => 0, 'pageviews' => (int)$iPageviews, 'bounces' => 0, 'duration' => 0];
  523. ksort($aRows);
  524. return $aRows;
  525. }
  526. /**
  527. * Top values of a dimension with visitors, sessions (session dimensions) and pageviews.
  528. */
  529. public function getTop($aFilter, $sDimension, $iLimit, $iStart = 0)
  530. {
  531. $sLimit = ' LIMIT ' . max(0, (int)$iStart) . ', ' . max(1, min(1000, (int)$iLimit));
  532. $aBindings = [];
  533. if(isset($this->_aEventDimensions[$sDimension])) {
  534. $sColumn = $this->_aEventDimensions[$sDimension];
  535. $sWhere = $this->_getEventsWhere($aFilter, $aBindings);
  536. // grouped by the column: events also have a `value` column, which an alias would lose to
  537. return $this->getAll("SELECT " . self::SQL_GROUP_WIDE . " `e`.`" . $sColumn . "` AS `value`, COUNT(DISTINCT `e`.`visitor`) AS `visitors`, COUNT(*) AS `pageviews` FROM `" . $this->_oConfig->CNF['TABLE_EVENTS'] . "` AS `e` WHERE " . $sWhere . " GROUP BY `e`.`" . $sColumn . "` ORDER BY `visitors` DESC, `pageviews` DESC" . $sLimit, $aBindings);
  538. }
  539. if(!isset($this->_aSessionDimensions[$sDimension]))
  540. return [];
  541. $sColumn = $this->_aSessionDimensions[$sDimension];
  542. $sWhere = $this->_getSessionsWhere($aFilter, $aBindings);
  543. return $this->getAll("SELECT `s`.`" . $sColumn . "` AS `value`, COUNT(DISTINCT `s`.`visitor`) AS `visitors`, COUNT(*) AS `sessions`, IFNULL(SUM(`s`.`pageviews`), 0) AS `pageviews`, IFNULL(SUM(`s`.`bounce`), 0) AS `bounces`, IFNULL(SUM(`s`.`duration`), 0) AS `duration` FROM " . $this->_getSessionsFrom() . " WHERE " . $sWhere . " GROUP BY `s`.`" . $sColumn . "` ORDER BY `visitors` DESC, `sessions` DESC" . $sLimit, $aBindings);
  544. }
  545. /**
  546. * How many different values a dimension has in the period, for paging.
  547. */
  548. public function getTopCount($aFilter, $sDimension)
  549. {
  550. $aBindings = [];
  551. if(isset($this->_aEventDimensions[$sDimension])) {
  552. $sWhere = $this->_getEventsWhere($aFilter, $aBindings);
  553. return (int)$this->getOne("SELECT COUNT(DISTINCT `e`.`" . $this->_aEventDimensions[$sDimension] . "`) FROM `" . $this->_oConfig->CNF['TABLE_EVENTS'] . "` AS `e` WHERE " . $sWhere, $aBindings);
  554. }
  555. if(!isset($this->_aSessionDimensions[$sDimension]))
  556. return 0;
  557. $sWhere = $this->_getSessionsWhere($aFilter, $aBindings);
  558. return (int)$this->getOne("SELECT COUNT(DISTINCT `s`.`" . $this->_aSessionDimensions[$sDimension] . "`) FROM " . $this->_getSessionsFrom() . " WHERE " . $sWhere, $aBindings);
  559. }
  560. /**
  561. * Values of any dimension with every metric it has, sorted by one of them, a page at a
  562. * time: the Query report and its CSV. Session dimensions get the summed session columns
  563. * (visitors, sessions, pageviews, bounces, duration), event dimensions what getPageStats
  564. * returns. Ties go by the number of rows, then by the value, so pages don't overlap.
  565. *
  566. * @param string $sMetric one of getBreakdownMetrics($sDimension); anything else sorts by the first
  567. * @param string $sDir ASC or DESC (anything else is DESC)
  568. * @param int $iMax the most rows one call returns
  569. * @param bool $bPathSessions pages get entries, bounces and exits (see getPageStats)
  570. */
  571. public function getBreakdown($aFilter, $sDimension, $sMetric, $sDir, $iLimit, $iStart = 0, $iMax = 1000, $bPathSessions = true)
  572. {
  573. $sDir = $sDir === 'ASC' ? 'ASC' : 'DESC';
  574. if(isset($this->_aEventDimensions[$sDimension]))
  575. return $this->getPageStats($aFilter, $sDimension, $iLimit, $iStart, isset($this->_aEventMetrics[$sMetric]) ? $sMetric : 'visitors', $sDir, $iMax, $bPathSessions);
  576. if(!isset($this->_aSessionDimensions[$sDimension]))
  577. return [];
  578. $sColumn = $this->_aSessionDimensions[$sDimension];
  579. $sOrder = isset($this->_aSessionMetrics[$sMetric]) ? $this->_aSessionMetrics[$sMetric] : $this->_aSessionMetrics['visitors'];
  580. $aBindings = [];
  581. $sWhere = $this->_getSessionsWhere($aFilter, $aBindings);
  582. return $this->getAll("SELECT `s`.`" . $sColumn . "` AS `value`, COUNT(DISTINCT `s`.`visitor`) AS `visitors`, COUNT(*) AS `sessions`, IFNULL(SUM(`s`.`pageviews`), 0) AS `pageviews`, IFNULL(SUM(`s`.`bounce`), 0) AS `bounces`, IFNULL(SUM(`s`.`duration`), 0) AS `duration` FROM " . $this->_getSessionsFrom() . " WHERE " . $sWhere . " GROUP BY `s`.`" . $sColumn . "` ORDER BY " . $sOrder . " " . $sDir . ", COUNT(*) " . $sDir . ", `s`.`" . $sColumn . "` ASC LIMIT " . max(0, (int)$iStart) . ", " . max(1, min((int)$iMax, (int)$iLimit)), $aBindings);
  583. }
  584. /**
  585. * The metrics of some values of a dimension per time bucket: rows with bucket, value and
  586. * the columns getBreakdown returns for that dimension (without entries and exits).
  587. * `value` is the asked value itself: the tables compare text without case and trailing
  588. * spaces, so a group's own spelling ('facebook' for 'Facebook') can differ per bucket.
  589. *
  590. * @param array $aValues the values to follow, a few at most
  591. * @see getSeries for the bucket parameters
  592. */
  593. public function getBreakdownSeries($aFilter, $sDimension, $aValues, $sBucket, $iOffset, $aStarts = [])
  594. {
  595. if(empty($aValues))
  596. return [];
  597. $aValues = array_values(array_map('strval', $aValues));
  598. $aBindings = [];
  599. if(isset($this->_aEventDimensions[$sDimension])) {
  600. $sColumn = '`e`.`' . $this->_aEventDimensions[$sDimension] . '`';
  601. $sWhere = $this->_getEventsWhere($aFilter, $aBindings);
  602. $sIn = $this->_getInSql($aValues, 'qv', $aBindings);
  603. $sValue = $this->_getValueCaseSql($sColumn, $aValues, $aBindings);
  604. $sBucketSql = $this->_getBucketSql('`e`.`ts`', $sBucket, $iOffset, $aStarts, $aBindings);
  605. // GROUP BY looks a name up in the table before the select list, and events have a
  606. // `value` column: grouped by that, every value of a bucket was one row
  607. $aRows = $this->getAll("SELECT " . $sBucketSql . " AS `bucket`, " . $sValue . " AS `asked`, COUNT(DISTINCT `e`.`visitor`) AS `visitors`, COUNT(*) AS `pageviews`, IFNULL(AVG(NULLIF(`e`.`engaged`, 0)), 0) / 1000 AS `engaged` FROM `" . $this->_oConfig->CNF['TABLE_EVENTS'] . "` AS `e` WHERE " . $sWhere . " AND " . $sColumn . " IN (" . $sIn . ") GROUP BY `bucket`, `asked` ORDER BY `bucket`", $aBindings);
  608. foreach($aRows as $i => $aRow) {
  609. $aRows[$i] = array_merge(['bucket' => $aRow['bucket'], 'value' => $aRow['asked']], $aRow);
  610. unset($aRows[$i]['asked']);
  611. }
  612. return $aRows;
  613. }
  614. if(!isset($this->_aSessionDimensions[$sDimension]))
  615. return [];
  616. $sColumn = '`s`.`' . $this->_aSessionDimensions[$sDimension] . '`';
  617. $sWhere = $this->_getSessionsWhere($aFilter, $aBindings);
  618. $sIn = $this->_getInSql($aValues, 'qv', $aBindings);
  619. $sValue = $this->_getValueCaseSql($sColumn, $aValues, $aBindings);
  620. $sBucketSql = $this->_getBucketSql('`s`.`started`', $sBucket, $iOffset, $aStarts, $aBindings);
  621. return $this->getAll("SELECT " . $sBucketSql . " AS `bucket`, " . $sValue . " AS `value`, COUNT(DISTINCT `s`.`visitor`) AS `visitors`, COUNT(*) AS `sessions`, IFNULL(SUM(`s`.`pageviews`), 0) AS `pageviews`, IFNULL(SUM(`s`.`bounce`), 0) AS `bounces`, IFNULL(SUM(`s`.`duration`), 0) AS `duration` FROM " . $this->_getSessionsFrom() . " WHERE " . $sWhere . " AND " . $sColumn . " IN (" . $sIn . ") GROUP BY `bucket`, `value` ORDER BY `bucket`", $aBindings);
  622. }
  623. /**
  624. * CASE expression that turns a column into the first of some values it equals, in the
  625. * column's collation.
  626. */
  627. protected function _getValueCaseSql($sColumn, $aValues, &$aBindings)
  628. {
  629. $sResult = 'CASE';
  630. foreach(array_values($aValues) as $i => $sValue) {
  631. $sResult .= ' WHEN ' . $sColumn . ' = :qw' . $i . ' THEN :qt' . $i;
  632. $aBindings['qw' . $i] = $sValue;
  633. $aBindings['qt' . $i] = $sValue;
  634. }
  635. return $sResult . ' END';
  636. }
  637. /**
  638. * Pages (or titles) with visitors, pageviews and average engaged time in seconds; pages
  639. * also get entries, bounces of the sessions that entered there, and exits.
  640. *
  641. * @param string $sMetric sort by this metric (visitors, pageviews, engaged), ties by the
  642. * row count and the value; empty for the usual order, visitors then pageviews
  643. * @param string $sDir ASC or DESC, with a metric
  644. * @param int $iMax the most rows one call returns
  645. * @param bool $bPathSessions pages get entries, bounces and exits; false skips the two
  646. * queries on visits when only the values and their own figures are needed
  647. */
  648. public function getPageStats($aFilter, $sDimension, $iLimit, $iStart = 0, $sMetric = '', $sDir = 'DESC', $iMax = 1000, $bPathSessions = true)
  649. {
  650. if(!isset($this->_aEventDimensions[$sDimension]))
  651. return [];
  652. $sColumn = $this->_aEventDimensions[$sDimension];
  653. $aBindings = [];
  654. $sWhere = $this->_getEventsWhere($aFilter, $aBindings);
  655. $sOrder = '`visitors` DESC, `pageviews` DESC';
  656. if(isset($this->_aEventMetrics[$sMetric])) {
  657. $sDir = $sDir === 'ASC' ? 'ASC' : 'DESC';
  658. $sOrder = $this->_aEventMetrics[$sMetric] . ' ' . $sDir . ', COUNT(*) ' . $sDir . ', `e`.`' . $sColumn . '` ASC';
  659. }
  660. $aRows = $this->getAllWithKey("SELECT " . self::SQL_GROUP_WIDE . " `e`.`" . $sColumn . "` AS `value`, COUNT(DISTINCT `e`.`visitor`) AS `visitors`, COUNT(*) AS `pageviews`, IFNULL(AVG(NULLIF(`e`.`engaged`, 0)), 0) / 1000 AS `engaged` FROM `" . $this->_oConfig->CNF['TABLE_EVENTS'] . "` AS `e` WHERE " . $sWhere . " GROUP BY `e`.`" . $sColumn . "` ORDER BY " . $sOrder . " LIMIT " . max(0, (int)$iStart) . ", " . max(1, min((int)$iMax, (int)$iLimit)), 'value', $aBindings);
  661. if(empty($aRows) || $sDimension != 'path' || !$bPathSessions)
  662. return array_values($aRows);
  663. foreach($aRows as $sPath => $aRow)
  664. $aRows[$sPath] = array_merge($aRow, ['entries' => 0, 'bounces' => 0, 'bounce_rate' => 0, 'exits' => 0]);
  665. // entries, bounces and exits come from the sessions that started in the period
  666. $aSessionFilter = $aFilter;
  667. unset($aSessionFilter['dims']['path']);
  668. foreach(['entry_path' => ['entries', 'bounces'], 'exit_path' => ['exits', '']] as $sSessionColumn => list($sCount, $sBounces)) {
  669. $aSessionBindings = [];
  670. $sSessionWhere = $this->_getSessionsWhere($aSessionFilter, $aSessionBindings);
  671. $aIn = [];
  672. foreach(array_keys($aRows) as $i => $sPath) {
  673. $aIn[] = ':p' . $i;
  674. $aSessionBindings['p' . $i] = (string)$sPath;
  675. }
  676. $aCounts = $this->getAllWithKey("SELECT `s`.`" . $sSessionColumn . "` AS `value`, COUNT(*) AS `cnt`, SUM(`s`.`bounce`) AS `bounces` FROM " . $this->_getSessionsFrom() . " WHERE " . $sSessionWhere . " AND `s`.`" . $sSessionColumn . "` IN (" . implode(', ', $aIn) . ") GROUP BY `s`.`" . $sSessionColumn . "`", 'value', $aSessionBindings);
  677. foreach($aCounts as $sPath => $aCount) {
  678. if(!isset($aRows[$sPath]))
  679. continue;
  680. $aRows[$sPath][$sCount] = (int)$aCount['cnt'];
  681. if($sBounces) {
  682. $aRows[$sPath][$sBounces] = (int)$aCount['bounces'];
  683. $aRows[$sPath]['bounce_rate'] = $aCount['cnt'] ? round(100 * $aCount['bounces'] / $aCount['cnt'], 1) : 0;
  684. }
  685. }
  686. }
  687. return array_values($aRows);
  688. }
  689. /**
  690. * Pageviews per minute since a time, keyed by the minute's start.
  691. */
  692. public function getPageviewsPerMinute($iSince, $sPlatform = '')
  693. {
  694. $aBindings = ['since' => (int)$iSince];
  695. $sWhere = '';
  696. if($sPlatform !== '') {
  697. $sWhere = ' AND `platform` = :platform';
  698. $aBindings['platform'] = $sPlatform;
  699. }
  700. return $this->getPairs("SELECT FLOOR(`ts` / 60) * 60 AS `minute`, COUNT(*) AS `pageviews` FROM `" . $this->_oConfig->CNF['TABLE_EVENTS'] . "` WHERE `ts` >= :since AND `type` = 'pageview'" . $sWhere . " GROUP BY `minute`", 'minute', 'pageviews', $aBindings);
  701. }
  702. /*
  703. * Behavior: events, sessions, members
  704. */
  705. /**
  706. * Events grouped by type, name and module, most frequent first.
  707. */
  708. public function getEventList($aFilter, $aTypes, $iLimit, $iStart = 0)
  709. {
  710. $aBindings = [];
  711. $sWhere = $this->_getAnyEventsWhere($aFilter, $aTypes, $aBindings);
  712. return $this->getAll("SELECT `e`.`type`, `e`.`name`, `e`.`module`, COUNT(*) AS `count`, COUNT(DISTINCT `e`.`visitor`) AS `visitors`, MAX(`e`.`ts`) AS `last` FROM `" . $this->_oConfig->CNF['TABLE_EVENTS'] . "` AS `e` WHERE " . $sWhere . " GROUP BY `e`.`type`, `e`.`name`, `e`.`module` ORDER BY `count` DESC LIMIT " . max(0, (int)$iStart) . ", " . max(1, min(1000, (int)$iLimit)), $aBindings);
  713. }
  714. public function getEventListCount($aFilter, $aTypes)
  715. {
  716. $aBindings = [];
  717. $sWhere = $this->_getAnyEventsWhere($aFilter, $aTypes, $aBindings);
  718. return (int)$this->getOne("SELECT COUNT(DISTINCT `e`.`type`, `e`.`name`, `e`.`module`) FROM `" . $this->_oConfig->CNF['TABLE_EVENTS'] . "` AS `e` WHERE " . $sWhere, $aBindings);
  719. }
  720. /**
  721. * One event's count and visitors per time bucket, per platform, and the properties of
  722. * its latest occurrences.
  723. */
  724. public function getEventSeries($aFilter, $aEvent, $sBucket, $iOffset, $aStarts = [])
  725. {
  726. $aBindings = [];
  727. $sWhere = $this->_getAnyEventsWhere($aFilter, [$aEvent['type']], $aBindings, $aEvent, $this->_isShortPeriod($aFilter));
  728. $sBucketSql = $this->_getBucketSql('`e`.`ts`', $sBucket, $iOffset, $aStarts, $aBindings);
  729. return $this->getAllWithKey("SELECT " . $sBucketSql . " AS `bucket`, COUNT(*) AS `count`, COUNT(DISTINCT `e`.`visitor`) AS `visitors` FROM `" . $this->_oConfig->CNF['TABLE_EVENTS'] . "` AS `e` WHERE " . $sWhere . " GROUP BY `bucket` ORDER BY `bucket`", 'bucket', $aBindings);
  730. }
  731. public function getEventPlatforms($aFilter, $aEvent)
  732. {
  733. $aBindings = [];
  734. $sWhere = $this->_getAnyEventsWhere($aFilter, [$aEvent['type']], $aBindings, $aEvent, $this->_isShortPeriod($aFilter));
  735. return $this->getAll("SELECT `e`.`platform`, COUNT(*) AS `count`, COUNT(DISTINCT `e`.`visitor`) AS `visitors` FROM `" . $this->_oConfig->CNF['TABLE_EVENTS'] . "` AS `e` WHERE " . $sWhere . " GROUP BY `e`.`platform` ORDER BY `count` DESC", $aBindings);
  736. }
  737. public function getEventProps($aFilter, $aEvent, $iLimit = 2000)
  738. {
  739. $aBindings = [];
  740. $sWhere = $this->_getAnyEventsWhere($aFilter, [$aEvent['type']], $aBindings, $aEvent, $this->_isShortPeriod($aFilter));
  741. return $this->getColumn("SELECT `e`.`props` FROM `" . $this->_oConfig->CNF['TABLE_EVENTS'] . "` AS `e` WHERE " . $sWhere . " AND `e`.`props` != '' ORDER BY `e`.`id` DESC LIMIT " . max(1, min(10000, (int)$iLimit)), $aBindings);
  742. }
  743. /**
  744. * Sessions that started in the period, latest first, optionally of one member.
  745. */
  746. public function getSessionList($aFilter, $iLimit, $iStart = 0, $iProfileId = 0)
  747. {
  748. $aBindings = [];
  749. $sWhere = $this->_getSessionsWhere($aFilter, $aBindings);
  750. if($iProfileId) {
  751. $sWhere .= ' AND `s`.`profile_id` = :profile';
  752. $aBindings['profile'] = (int)$iProfileId;
  753. }
  754. return $this->getAll("SELECT `s`.* FROM " . $this->_getSessionsFrom() . " WHERE " . $sWhere . " ORDER BY `s`.`started` DESC LIMIT " . max(0, (int)$iStart) . ", " . max(1, min(1000, (int)$iLimit)), $aBindings);
  755. }
  756. public function getSessionListCount($aFilter, $iProfileId = 0)
  757. {
  758. $aBindings = [];
  759. $sWhere = $this->_getSessionsWhere($aFilter, $aBindings);
  760. if($iProfileId) {
  761. $sWhere .= ' AND `s`.`profile_id` = :profile';
  762. $aBindings['profile'] = (int)$iProfileId;
  763. }
  764. return (int)$this->getOne("SELECT COUNT(*) FROM " . $this->_getSessionsFrom() . " WHERE " . $sWhere, $aBindings);
  765. }
  766. public function getSession($iId)
  767. {
  768. return $this->getRow("SELECT * FROM `" . $this->_oConfig->CNF['TABLE_SESSIONS'] . "` WHERE `id` = :id", ['id' => (int)$iId]);
  769. }
  770. public function getSessionEvents($iId, $iLimit = 500)
  771. {
  772. return $this->getAll("SELECT `ts`, `type`, `name`, `path`, `title`, `module`, `object_id`, `value`, `engaged`, `props` FROM `" . $this->_oConfig->CNF['TABLE_EVENTS'] . "` WHERE `session_id` = :id ORDER BY `id` LIMIT " . max(1, min(5000, (int)$iLimit)), ['id' => (int)$iId]);
  773. }
  774. /**
  775. * Members who visited in the period with their visits, pageviews, time and platforms,
  776. * the latest visitor first.
  777. */
  778. public function getMemberList($aFilter, $iLimit, $iStart = 0)
  779. {
  780. $aBindings = [];
  781. $sWhere = $this->_getSessionsWhere($aFilter, $aBindings) . ' AND `s`.`profile_id` > 0';
  782. // the page of members first, from the `profile_id` key alone (it holds the start, the
  783. // last hit and the platform), then their figures: all of them at once read every
  784. // visit of every member in the period (9 s for 375,000 visits at 90 days)
  785. $aIds = $this->getColumn("SELECT `s`.`profile_id` FROM " . $this->_getSessionsFrom() . " WHERE " . $sWhere . " GROUP BY `s`.`profile_id` ORDER BY MAX(`s`.`last_seen`) DESC, `s`.`profile_id` DESC LIMIT " . max(0, (int)$iStart) . ", " . max(1, min(1000, (int)$iLimit)), $aBindings);
  786. if(empty($aIds))
  787. return [];
  788. $sIn = $this->_getInSql(array_map('intval', $aIds), 'm', $aBindings);
  789. $aRows = $this->getAllWithKey("SELECT `s`.`profile_id`, COUNT(*) AS `sessions`, SUM(`s`.`pageviews`) AS `pageviews`, SUM(`s`.`duration`) AS `duration`, MIN(`s`.`started`) AS `first`, MAX(`s`.`last_seen`) AS `last`, GROUP_CONCAT(DISTINCT `s`.`platform` ORDER BY `s`.`platform`) AS `platforms`, SUBSTRING_INDEX(GROUP_CONCAT(`s`.`country` ORDER BY `s`.`last_seen` DESC), ',', 1) AS `country` FROM " . $this->_getSessionsFrom() . " WHERE " . $sWhere . " AND `s`.`profile_id` IN (" . $sIn . ") GROUP BY `s`.`profile_id`", 'profile_id', $aBindings);
  790. $aResult = [];
  791. foreach($aIds as $iId)
  792. if(isset($aRows[$iId]))
  793. $aResult[] = $aRows[$iId];
  794. return $aResult;
  795. }
  796. public function getMemberListCount($aFilter)
  797. {
  798. $aBindings = [];
  799. $sWhere = $this->_getSessionsWhere($aFilter, $aBindings);
  800. return (int)$this->getOne("SELECT COUNT(DISTINCT `s`.`profile_id`) FROM " . $this->_getSessionsFrom() . " WHERE " . $sWhere . " AND `s`.`profile_id` > 0", $aBindings);
  801. }
  802. /**
  803. * A member's totals over all the data kept, and per platform.
  804. */
  805. public function getMemberTotals($iProfileId)
  806. {
  807. $aBindings = ['profile' => (int)$iProfileId];
  808. $sTable = $this->_oConfig->CNF['TABLE_SESSIONS'];
  809. return [
  810. 'totals' => $this->getRow("SELECT COUNT(*) AS `sessions`, IFNULL(SUM(`pageviews`), 0) AS `pageviews`, IFNULL(SUM(`duration`), 0) AS `duration`, MIN(`started`) AS `first`, MAX(`last_seen`) AS `last` FROM `" . $sTable . "` WHERE `profile_id` = :profile", $aBindings),
  811. 'platforms' => $this->getPairs("SELECT `platform`, COUNT(*) AS `sessions` FROM `" . $sTable . "` WHERE `profile_id` = :profile GROUP BY `platform` ORDER BY `sessions` DESC", 'platform', 'sessions', $aBindings),
  812. ];
  813. }
  814. /**
  815. * A member's events and site actions in the period, most frequent first.
  816. */
  817. public function getMemberActions($iProfileId, $aFilter, $iLimit = 20)
  818. {
  819. $aBindings = [];
  820. $sWhere = $this->_getAnyEventsWhere(array_merge($aFilter, ['dims' => []]), [BX_ANALYTICS_HIT_EVENT, BX_ANALYTICS_HIT_UNA], $aBindings);
  821. $aBindings['profile'] = (int)$iProfileId;
  822. return $this->getAll("SELECT `e`.`type`, `e`.`name`, `e`.`module`, COUNT(*) AS `count`, MAX(`e`.`ts`) AS `last` FROM `" . $this->_oConfig->CNF['TABLE_EVENTS'] . "` AS `e` WHERE " . $sWhere . " AND `e`.`profile_id` = :profile GROUP BY `e`.`type`, `e`.`name`, `e`.`module` ORDER BY `count` DESC LIMIT " . max(1, min(100, (int)$iLimit)), $aBindings);
  823. }
  824. /*
  825. * Conversion: journeys, retention
  826. */
  827. /**
  828. * Each visit's pages in order, for journeys, a page reloaded in place once: [session id
  829. * => [path, ...]] in session order, from at most `$iLimit` pageviews (the latest sessions
  830. * win when there are more), and how many pageviews were read.
  831. *
  832. * The rows are folded into the visits as they are read, and every path is kept once and
  833. * shared: a list of rows of the same size took over 100 MB.
  834. *
  835. * @return array sequences, rows
  836. */
  837. public function getPageSequences($aFilter, $iLimit = 200000)
  838. {
  839. $aBindings = [];
  840. $sWhere = $this->_getEventsWhere($aFilter, $aBindings, true);
  841. // one direction, the order of the `session_id` key (session, time, id), read backwards
  842. // and stopped at the limit: sorted the other way within a visit, the server sorted every
  843. // pageview of the period first (5 s for 7 million at 90 days)
  844. $oStatement = $this->prepare("SELECT `e`.`session_id`, `e`.`path` FROM `" . $this->_oConfig->CNF['TABLE_EVENTS'] . "` AS `e` WHERE " . $sWhere . " ORDER BY `e`.`session_id` DESC, `e`.`ts` DESC, `e`.`id` DESC LIMIT " . max(1, min(1000000, (int)$iLimit)));
  845. if(!$oStatement || !$this->res($oStatement, $aBindings))
  846. return ['sequences' => [], 'rows' => 0];
  847. $aPaths = [];
  848. $aSequences = [];
  849. $iRows = 0;
  850. while(($aRow = $oStatement->fetch(PDO::FETCH_NUM)) !== false) {
  851. $iRows++;
  852. $sPath = (string)$aRow[1];
  853. if(!isset($aPaths[$sPath]))
  854. $aPaths[$sPath] = $sPath;
  855. $sPath = $aPaths[$sPath];
  856. // backwards: a visit's last page comes first
  857. $iSession = (int)$aRow[0];
  858. if(!isset($aSequences[$iSession]))
  859. $aSequences[$iSession] = [$sPath];
  860. elseif(end($aSequences[$iSession]) !== $sPath)
  861. $aSequences[$iSession][] = $sPath;
  862. }
  863. $oStatement->closeCursor();
  864. foreach($aSequences as $iSession => $aSequence)
  865. $aSequences[$iSession] = array_reverse($aSequence);
  866. return ['sequences' => array_reverse($aSequences, true), 'rows' => $iRows];
  867. }
  868. /**
  869. * Weekly cohorts counted in the database, in one pass over the visits kept: visitors
  870. * whose first visit (ever kept) falls in the period, by the week of that visit and the
  871. * weeks they visited in, as a bit mask (bit n: week n of the period). Members only, or
  872. * every visitor id. The period starts on a week boundary and weeks are 7 days long, as
  873. * the week buckets of the reports. A visitor's own rows never reach PHP.
  874. *
  875. * The visits are read in the order of the `visitor` key, which holds every column used
  876. * here, so the grouping needs neither a sort nor a temporary table.
  877. *
  878. * @param int $iFrom the first week's start
  879. * @return array rows of cohort (the week of the first visit, 0 for the period's first),
  880. * weeks (the bit mask) and visitors
  881. */
  882. public function getRetentionCohorts($iFrom, $iTo, $sPlatform, $bMembersOnly)
  883. {
  884. $aBindings = ['to' => (int)$iTo, 'from_first' => (int)$iFrom, 'from_in' => (int)$iFrom, 'from_week' => (int)$iFrom, 'from_cohort' => (int)$iFrom];
  885. $sWhere = '`s`.`started` < :to';
  886. if($bMembersOnly)
  887. $sWhere .= ' AND `s`.`profile_id` > 0';
  888. if($sPlatform !== '') {
  889. $sWhere .= ' AND `s`.`platform` = :platform';
  890. $aBindings['platform'] = $sPlatform;
  891. }
  892. return $this->getAll("SELECT FLOOR((`f`.`first` - :from_cohort) / 604800) AS `cohort`, `f`.`weeks`, COUNT(*) AS `visitors`
  893. FROM (SELECT MIN(`s`.`started`) AS `first`, BIT_OR(IF(`s`.`started` >= :from_in, 1 << FLOOR((`s`.`started` - :from_week) / 604800), 0)) AS `weeks` FROM " . $this->_getSessionsFrom() . " WHERE " . $sWhere . " GROUP BY `s`.`visitor` HAVING `first` >= :from_first) AS `f`
  894. GROUP BY `cohort`, `f`.`weeks`", $aBindings);
  895. }
  896. /*
  897. * Goals and funnels
  898. */
  899. /**
  900. * Goals in their order, the first `$iLimit` only when given.
  901. */
  902. public function getGoals($iLimit = 0)
  903. {
  904. return $this->getAll("SELECT * FROM `" . $this->_oConfig->CNF['TABLE_GOALS'] . "` ORDER BY `order`, `id`" . ((int)$iLimit > 0 ? " LIMIT " . (int)$iLimit : ''));
  905. }
  906. public function getGoalsCount()
  907. {
  908. return (int)$this->getOne("SELECT COUNT(*) FROM `" . $this->_oConfig->CNF['TABLE_GOALS'] . "`");
  909. }
  910. public function getGoal($iId)
  911. {
  912. return $this->getRow("SELECT * FROM `" . $this->_oConfig->CNF['TABLE_GOALS'] . "` WHERE `id` = :id", ['id' => (int)$iId]);
  913. }
  914. public function saveGoal($iId, $aGoal)
  915. {
  916. $sTable = $this->_oConfig->CNF['TABLE_GOALS'];
  917. $aBindings = ['name' => $aGoal['name'], 'type' => $aGoal['type'], 'pattern' => $aGoal['pattern']];
  918. if($iId)
  919. return $this->query("UPDATE `" . $sTable . "` SET `name` = :name, `type` = :type, `pattern` = :pattern WHERE `id` = :id", $aBindings + ['id' => (int)$iId]) !== false ? (int)$iId : 0;
  920. if(!$this->query("INSERT INTO `" . $sTable . "` (`name`, `type`, `pattern`, `added`, `order`) VALUES (:name, :type, :pattern, UNIX_TIMESTAMP(), (SELECT IFNULL(MAX(`o`.`order`), 0) + 1 FROM `" . $sTable . "` AS `o`))", $aBindings))
  921. return 0;
  922. return (int)$this->lastId();
  923. }
  924. public function deleteGoal($iId)
  925. {
  926. return (int)$this->query("DELETE FROM `" . $this->_oConfig->CNF['TABLE_GOALS'] . "` WHERE `id` = :id", ['id' => (int)$iId]) > 0;
  927. }
  928. public function getFunnels()
  929. {
  930. return $this->getAll("SELECT * FROM `" . $this->_oConfig->CNF['TABLE_FUNNELS'] . "` ORDER BY `order`, `id`");
  931. }
  932. public function getFunnel($iId)
  933. {
  934. return $this->getRow("SELECT * FROM `" . $this->_oConfig->CNF['TABLE_FUNNELS'] . "` WHERE `id` = :id", ['id' => (int)$iId]);
  935. }
  936. public function saveFunnel($iId, $sName, $aSteps)
  937. {
  938. $sTable = $this->_oConfig->CNF['TABLE_FUNNELS'];
  939. $aBindings = ['name' => $sName, 'steps' => json_encode(array_values($aSteps), JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES)];
  940. if($iId)
  941. return $this->query("UPDATE `" . $sTable . "` SET `name` = :name, `steps` = :steps WHERE `id` = :id", $aBindings + ['id' => (int)$iId]) !== false ? (int)$iId : 0;
  942. if(!$this->query("INSERT INTO `" . $sTable . "` (`name`, `steps`, `added`, `order`) VALUES (:name, :steps, UNIX_TIMESTAMP(), (SELECT IFNULL(MAX(`o`.`order`), 0) + 1 FROM `" . $sTable . "` AS `o`))", $aBindings))
  943. return 0;
  944. return (int)$this->lastId();
  945. }
  946. public function deleteFunnel($iId)
  947. {
  948. return (int)$this->query("DELETE FROM `" . $this->_oConfig->CNF['TABLE_FUNNELS'] . "` WHERE `id` = :id", ['id' => (int)$iId]) > 0;
  949. }
  950. /**
  951. * Visits and visitors that reached a goal (a page or an event) in the period. A site action
  952. * outside a visit (session_id 0: a payment webhook, cron) is not a visit, but its visitor
  953. * reached the goal.
  954. */
  955. public function getGoalTotals($aFilter, $aGoal)
  956. {
  957. $aBindings = [];
  958. $sWhere = $this->_getAnyEventsWhere($aFilter, [BX_ANALYTICS_HIT_PAGEVIEW, BX_ANALYTICS_HIT_EVENT, BX_ANALYTICS_HIT_UNA], $aBindings, null, $this->_isShortPeriod($aFilter)) . ' AND ' . $this->_getMatchSql($aGoal, $aBindings, 'g');
  959. // nothing but the columns of the `path` key (or `name` and the visit), so a page goal
  960. // reads no row
  961. return $this->getRow("SELECT COUNT(DISTINCT NULLIF(`e`.`session_id`, 0)) AS `sessions`, COUNT(DISTINCT `e`.`visitor`) AS `visitors` FROM `" . $this->_oConfig->CNF['TABLE_EVENTS'] . "` AS `e` WHERE " . $sWhere, $aBindings);
  962. }
  963. /**
  964. * Visits that reached each step of a funnel in order, counted in the database in one
  965. * query. A visit reaches the first step the first time it matches it in the period (and
  966. * the filters), and each next step the first time it matches it after the event that
  967. * reached the step before; one event counts for one step. One row per visit and step,
  968. * never one per event, reaches the server's temporary tables, and none reaches PHP.
  969. *
  970. * Each step joins the visits of the step before: [session_id, t0, k0, ..., t(n-1), k(n-1)]
  971. * with tn and kn, NULL when the visit didn't reach step n (and so no step after it). kn
  972. * orders the events: by the second they happened, then by the order they were stored
  973. * (ids follow the client's order in a request), so two hits in one second keep their order.
  974. *
  975. * @param array $aSteps steps of type and pattern, see _getMatchSql
  976. * @return array visits per step, in step order
  977. */
  978. public function getFunnelCounts($aFilter, $aSteps)
  979. {
  980. $aSteps = array_values($aSteps);
  981. if(empty($aSteps))
  982. return [];
  983. $sTable = $this->_oConfig->CNF['TABLE_EVENTS'];
  984. $aBindings = [];
  985. $sWhere = $this->_getAnyEventsWhere($aFilter, [BX_ANALYTICS_HIT_PAGEVIEW, BX_ANALYTICS_HIT_EVENT, BX_ANALYTICS_HIT_UNA], $aBindings, null, $this->_isShortPeriod($aFilter)) . ' AND ' . $this->_getMatchSql($aSteps[0], $aBindings, 's0') . ' AND `e`.`session_id` > 0';
  986. // 1e20 is past the id range: the second first, then the id
  987. $sKey = "MIN(CAST(`e`.`ts` AS DECIMAL(31,0)) * 100000000000000000000 + `e`.`id`)";
  988. $sSql = "SELECT `e`.`session_id`, MIN(`e`.`ts`) AS `t0`, " . $sKey . " AS `k0` FROM `" . $sTable . "` AS `e` WHERE " . $sWhere . " GROUP BY `e`.`session_id`";
  989. // the filters hold for every event of a visit that matched them, but the period
  990. $aCounts = ['COUNT(*) AS `c0`'];
  991. $aBefore = ['`p`.`t0`', '`p`.`k0`'];
  992. for($i = 1; $i < count($aSteps); $i++) {
  993. $aBindings['to' . $i] = (int)$aFilter['to'];
  994. $sSql = "SELECT `p`.`session_id`, " . implode(', ', $aBefore) . ", MIN(`e`.`ts`) AS `t" . $i . "`, " . $sKey . " AS `k" . $i . "` FROM (" . $sSql . ") AS `p` LEFT JOIN `" . $sTable . "` AS `e` ON `e`.`session_id` = `p`.`session_id` AND `e`.`ts` >= `p`.`t" . ($i - 1) . "` AND `e`.`ts` < :to" . $i . " AND CAST(`e`.`ts` AS DECIMAL(31,0)) * 100000000000000000000 + `e`.`id` > `p`.`k" . ($i - 1) . "` AND " . $this->_getMatchSql($aSteps[$i], $aBindings, 's' . $i) . " GROUP BY `p`.`session_id`, " . implode(', ', $aBefore);
  995. $aCounts[] = 'COUNT(`f`.`t' . $i . '`) AS `c' . $i . '`';
  996. $aBefore[] = '`p`.`t' . $i . '`';
  997. $aBefore[] = '`p`.`k' . $i . '`';
  998. }
  999. $aRow = $this->getRow("SELECT " . implode(', ', $aCounts) . " FROM (" . $sSql . ") AS `f`", $aBindings);
  1000. $aResult = [];
  1001. foreach(array_keys($aSteps) as $i)
  1002. $aResult[] = isset($aRow['c' . $i]) ? (int)$aRow['c' . $i] : 0;
  1003. return $aResult;
  1004. }
  1005. /**
  1006. * SQL that matches events to a goal or a funnel step: a page path (exact, or with * for
  1007. * any characters) or an event name (custom event or site action, `name@module` for one module).
  1008. */
  1009. protected function _getMatchSql($aMatch, &$aBindings, $sPrefix)
  1010. {
  1011. $sPattern = (string)$aMatch['pattern'];
  1012. if($aMatch['type'] == 'page') {
  1013. $aBindings[$sPrefix . '_pattern'] = str_replace('*', '%', addcslashes($sPattern, '\\%_'));
  1014. return "`e`.`type` = 'pageview' AND `e`.`path` LIKE :" . $sPrefix . "_pattern";
  1015. }
  1016. $aParts = explode('@', $sPattern, 2);
  1017. $aBindings[$sPrefix . '_name'] = $aParts[0];
  1018. $sSql = "`e`.`type` IN ('event', 'una') AND `e`.`name` = :" . $sPrefix . "_name";
  1019. if(isset($aParts[1]) && $aParts[1] !== '') {
  1020. $aBindings[$sPrefix . '_module'] = $aParts[1];
  1021. $sSql .= " AND `e`.`module` = :" . $sPrefix . "_module";
  1022. }
  1023. return $sSql;
  1024. }
  1025. /**
  1026. * WHERE for events (alias `e`) of some types in the period, optionally one event
  1027. * (type, name, module), with session filters through the event's session.
  1028. */
  1029. protected function _getAnyEventsWhere($aFilter, $aTypes, &$aBindings, $aEvent = null, $bCorrelated = false)
  1030. {
  1031. $aSql = ['`e`.`ts` >= :from', '`e`.`ts` < :to'];
  1032. $aBindings['from'] = (int)$aFilter['from'];
  1033. $aBindings['to'] = (int)$aFilter['to'];
  1034. $aIn = [];
  1035. foreach(array_values($aTypes) as $i => $sType) {
  1036. $aIn[] = ':t' . $i;
  1037. $aBindings['t' . $i] = $sType;
  1038. }
  1039. $aSql[] = '`e`.`type` IN (' . implode(', ', $aIn) . ')';
  1040. if($aEvent !== null) {
  1041. $aSql[] = '`e`.`name` = :ev_name';
  1042. $aSql[] = '`e`.`module` = :ev_module';
  1043. $aBindings['ev_name'] = $aEvent['name'];
  1044. $aBindings['ev_module'] = $aEvent['module'];
  1045. }
  1046. if($aFilter['platform'] !== '') {
  1047. $aSql[] = '`e`.`platform` = :platform';
  1048. $aBindings['platform'] = $aFilter['platform'];
  1049. }
  1050. // session dimensions go through the session; each page filter stands on its own
  1051. $aThrough = [];
  1052. foreach($aFilter['dims'] as $sName => $sValue) {
  1053. if(isset($this->_aSessionDimensions[$sName])) {
  1054. $aThrough[] = '`sf`.`' . $this->_aSessionDimensions[$sName] . '` = :d_' . $sName;
  1055. $aBindings['d_' . $sName] = $sValue;
  1056. }
  1057. elseif(isset($this->_aEventDimensions[$sName])) {
  1058. $aSql[] = $this->_getPageVisitsInSql('`e`.`session_id`', ['`ef`.`' . $this->_aEventDimensions[$sName] . '` = :d_' . $sName], $aFilter, $aBindings, '_' . $sName);
  1059. $aBindings['d_' . $sName] = $sValue;
  1060. }
  1061. }
  1062. if(!empty($aThrough))
  1063. $aSql[] = $this->_getVisitsInSql('`e`.`session_id`', $aThrough, $aFilter, $aBindings, $bCorrelated);
  1064. return implode(' AND ', $aSql);
  1065. }
  1066. /**
  1067. * Bot totals in a period: hits and distinct bots, optionally for some categories.
  1068. */
  1069. public function getBotTotals($aFilter, $aCategories = [])
  1070. {
  1071. $aBindings = [];
  1072. $sWhere = $this->_getBotsWhere($aFilter, $aCategories, $aBindings);
  1073. return $this->getRow("SELECT IFNULL(SUM(`hits`), 0) AS `hits`, COUNT(DISTINCT `bot`) AS `bots` FROM `" . $this->_oConfig->CNF['TABLE_BOTS'] . "` WHERE " . $sWhere, $aBindings);
  1074. }
  1075. /**
  1076. * Bot hits per day and category.
  1077. */
  1078. public function getBotSeries($aFilter)
  1079. {
  1080. $aBindings = [];
  1081. $sWhere = $this->_getBotsWhere($aFilter, [], $aBindings);
  1082. return $this->getAll("SELECT `day`, `category`, SUM(`hits`) AS `hits` FROM `" . $this->_oConfig->CNF['TABLE_BOTS'] . "` WHERE " . $sWhere . " GROUP BY `day`, `category` ORDER BY `day`", $aBindings);
  1083. }
  1084. /**
  1085. * Bots with their hits, last visit and platforms, busiest first.
  1086. */
  1087. public function getBotList($aFilter, $aCategories, $iLimit, $iStart = 0)
  1088. {
  1089. $aBindings = [];
  1090. $sWhere = $this->_getBotsWhere($aFilter, $aCategories, $aBindings);
  1091. return $this->getAll("SELECT `bot`, `category`, SUM(`hits`) AS `hits`, MAX(`last_seen`) AS `last_seen`, GROUP_CONCAT(DISTINCT `platform` ORDER BY `platform`) AS `platforms` FROM `" . $this->_oConfig->CNF['TABLE_BOTS'] . "` WHERE " . $sWhere . " GROUP BY `bot`, `category` ORDER BY `hits` DESC LIMIT " . max(0, (int)$iStart) . ", " . max(1, min(1000, (int)$iLimit)), $aBindings);
  1092. }
  1093. public function getBotListCount($aFilter, $aCategories)
  1094. {
  1095. $aBindings = [];
  1096. $sWhere = $this->_getBotsWhere($aFilter, $aCategories, $aBindings);
  1097. return (int)$this->getOne("SELECT COUNT(DISTINCT `bot`, `category`) FROM `" . $this->_oConfig->CNF['TABLE_BOTS'] . "` WHERE " . $sWhere, $aBindings);
  1098. }
  1099. /**
  1100. * Pages bots asked for most.
  1101. */
  1102. public function getBotPages($aFilter, $aCategories, $iLimit)
  1103. {
  1104. $aBindings = [];
  1105. $sWhere = $this->_getBotsWhere($aFilter, $aCategories, $aBindings, false);
  1106. return $this->getAll("SELECT `path`, SUM(`hits`) AS `hits` FROM `" . $this->_oConfig->CNF['TABLE_BOT_PAGES'] . "` WHERE " . $sWhere . " GROUP BY `path_hash`, `path` ORDER BY `hits` DESC LIMIT " . max(1, min(100, (int)$iLimit)), $aBindings);
  1107. }
  1108. /**
  1109. * Bot rows are per UTC day (the collector's gmdate), so a period in another timezone
  1110. * reads every UTC day it touches: from the UTC day of its first second to the UTC day
  1111. * of its last one. East of UTC that starts a day before the local first day, west of
  1112. * UTC it ends a day after the local last one. Reports chart these same days.
  1113. */
  1114. public function getBotDays($aFilter)
  1115. {
  1116. $iFrom = (int)$aFilter['from'];
  1117. return [
  1118. 'from' => gmdate('Y-m-d', $iFrom),
  1119. 'to' => gmdate('Y-m-d', max($iFrom, (int)$aFilter['to'] - 1)),
  1120. ];
  1121. }
  1122. /**
  1123. * Bot rows are per UTC day: the period's UTC days (getBotDays), its platform (the pages
  1124. * table has none) and the categories.
  1125. */
  1126. protected function _getBotsWhere($aFilter, $aCategories, &$aBindings, $bPlatform = true)
  1127. {
  1128. $aDays = $this->getBotDays($aFilter);
  1129. $aSql = ['`day` >= :day_from', '`day` <= :day_to'];
  1130. $aBindings['day_from'] = $aDays['from'];
  1131. $aBindings['day_to'] = $aDays['to'];
  1132. if($bPlatform && $aFilter['platform'] !== '') {
  1133. $aSql[] = '`platform` = :platform';
  1134. $aBindings['platform'] = $aFilter['platform'];
  1135. }
  1136. if(!empty($aCategories)) {
  1137. $aIn = [];
  1138. foreach(array_values($aCategories) as $i => $sCategory) {
  1139. $aIn[] = ':c' . $i;
  1140. $aBindings['c' . $i] = $sCategory;
  1141. }
  1142. $aSql[] = '`category` IN (' . implode(', ', $aIn) . ')';
  1143. }
  1144. return implode(' AND ', $aSql);
  1145. }
  1146. /**
  1147. * Visitors per rounded location (sessions with coordinates from the edge), for the map.
  1148. */
  1149. public function getMapPoints($aFilter, $iLimit = 500)
  1150. {
  1151. $aBindings = [];
  1152. $sWhere = $this->_getSessionsWhere($aFilter, $aBindings);
  1153. return $this->getAll("SELECT `s`.`lat`, `s`.`lng`, MAX(`s`.`city`) AS `city`, MAX(`s`.`country`) AS `country`, COUNT(DISTINCT `s`.`visitor`) AS `visitors` FROM " . $this->_getSessionsFrom() . " WHERE " . $sWhere . " AND `s`.`lat` IS NOT NULL GROUP BY `s`.`lat`, `s`.`lng` ORDER BY `visitors` DESC LIMIT " . max(1, min(2000, (int)$iLimit)), $aBindings);
  1154. }
  1155. /**
  1156. * Visitors active since a time, per platform.
  1157. */
  1158. public function getOnline($iSince)
  1159. {
  1160. return $this->getPairs("SELECT `platform`, COUNT(DISTINCT `visitor`) AS `visitors` FROM `" . $this->_oConfig->CNF['TABLE_SESSIONS'] . "` WHERE `last_seen` >= :since GROUP BY `platform`", 'platform', 'visitors', ['since' => (int)$iSince]);
  1161. }
  1162. /**
  1163. * Latest pageviews with their session's place and device, for the realtime list.
  1164. */
  1165. public function getRecentPageviews($iSince, $iLimit, $sPlatform = '')
  1166. {
  1167. $aBindings = ['since' => (int)$iSince];
  1168. $sWhere = '';
  1169. if($sPlatform !== '') {
  1170. $sWhere = ' AND `e`.`platform` = :platform';
  1171. $aBindings['platform'] = $sPlatform;
  1172. }
  1173. return $this->getAll("SELECT `e`.`ts`, `e`.`platform`, `e`.`path`, `e`.`title`, `e`.`profile_id`, `s`.`country`, `s`.`city`, `s`.`device`, `s`.`browser`, `s`.`os` FROM `" . $this->_oConfig->CNF['TABLE_EVENTS'] . "` AS `e` INNER JOIN `" . $this->_oConfig->CNF['TABLE_SESSIONS'] . "` AS `s` ON `s`.`id` = `e`.`session_id` WHERE `e`.`ts` >= :since AND `e`.`type` = 'pageview'" . $sWhere . " ORDER BY `e`.`id` DESC LIMIT " . max(1, min(100, (int)$iLimit)), $aBindings);
  1174. }
  1175. /**
  1176. * Whether anything was ever collected, per platform: the setup check in Studio.
  1177. */
  1178. public function getLastHits($iUntil)
  1179. {
  1180. return $this->getPairs("SELECT `platform`, MAX(`last_seen`) AS `last` FROM `" . $this->_oConfig->CNF['TABLE_SESSIONS'] . "` WHERE `last_seen` <= :until GROUP BY `platform`", 'platform', 'last', ['until' => (int)$iUntil]);
  1181. }
  1182. /**
  1183. * Hits in a time range per platform, from the index on events: platform => [hits, last].
  1184. */
  1185. public function getRecentHits($iSince, $iUntil)
  1186. {
  1187. return $this->getAllWithKey("SELECT `platform`, COUNT(*) AS `hits`, MAX(`ts`) AS `last` FROM `" . $this->_oConfig->CNF['TABLE_EVENTS'] . "` WHERE `ts` >= :since AND `ts` <= :until GROUP BY `platform`", 'platform', ['since' => (int)$iSince, 'until' => (int)$iUntil]);
  1188. }
  1189. /**
  1190. * The latest hit of a platform up to a time, visit or not (the server's own hits and
  1191. * actions outside a visit keep none): one lookup in the platform index.
  1192. */
  1193. public function getLastEvent($sPlatform, $iUntil)
  1194. {
  1195. return (int)$this->getOne("SELECT MAX(`ts`) FROM `" . $this->_oConfig->CNF['TABLE_EVENTS'] . "` WHERE `platform` = :platform AND `ts` <= :until", ['until' => (int)$iUntil, 'platform' => $sPlatform]);
  1196. }
  1197. /**
  1198. * Visitors in visits active in a time range, per platform.
  1199. */
  1200. public function getActiveVisitors($iSince, $iUntil)
  1201. {
  1202. return $this->getPairs("SELECT `platform`, COUNT(DISTINCT `visitor`) AS `visitors` FROM `" . $this->_oConfig->CNF['TABLE_SESSIONS'] . "` WHERE `last_seen` >= :since AND `started` <= :until GROUP BY `platform`", 'platform', 'visitors', ['since' => (int)$iSince, 'until' => (int)$iUntil]);
  1203. }
  1204. /**
  1205. * Visits started in a time range, per platform.
  1206. */
  1207. public function getStartedSessions($iSince, $iUntil)
  1208. {
  1209. return $this->getPairs("SELECT `platform`, COUNT(*) AS `sessions` FROM `" . $this->_oConfig->CNF['TABLE_SESSIONS'] . "` WHERE `started` >= :since AND `started` <= :until GROUP BY `platform`", 'platform', 'sessions', ['since' => (int)$iSince, 'until' => (int)$iUntil]);
  1210. }
  1211. /**
  1212. * App versions in visits started in a time range, as rows of platform and app_version.
  1213. */
  1214. public function getAppVersions($iSince, $iUntil, $aPlatforms)
  1215. {
  1216. if(empty($aPlatforms))
  1217. return [];
  1218. $aBindings = ['since' => (int)$iSince, 'until' => (int)$iUntil];
  1219. $sIn = $this->_getInSql($aPlatforms, 'platform', $aBindings);
  1220. return $this->getAll("SELECT DISTINCT `platform`, `app_version` FROM `" . $this->_oConfig->CNF['TABLE_SESSIONS'] . "` WHERE `started` >= :since AND `started` <= :until AND `platform` IN (" . $sIn . ") AND `app_version` <> '' LIMIT 1000", $aBindings);
  1221. }
  1222. /*
  1223. * Studio tools
  1224. */
  1225. /**
  1226. * Site pages with active analytics blocks: the module's own page, builder copies on any
  1227. * other page, and pages made with the Dashboards tool. Studio's own pages are left out.
  1228. * Each page with its number of blocks and how many of them NEO renders (active_api).
  1229. */
  1230. public function getPagesWithBlocks($sModule)
  1231. {
  1232. return $this->getAll("SELECT `p`.`object`, `p`.`uri`, `p`.`url`, `p`.`title`, `p`.`module`, `p`.`deletable`, COUNT(`b`.`id`) AS `blocks`, SUM(`b`.`active_api`) AS `api_blocks`
  1233. FROM `sys_pages_blocks` AS `b`
  1234. INNER JOIN `sys_objects_page` AS `p` ON `p`.`object` = `b`.`object`
  1235. WHERE `b`.`module` = :module AND `b`.`type` = 'service' AND `b`.`active` = 1 AND `b`.`object` <> '' AND `b`.`object` NOT LIKE 'sys\_std\_%'
  1236. GROUP BY `p`.`object`, `p`.`uri`, `p`.`url`, `p`.`title`, `p`.`module`, `p`.`deletable`
  1237. ORDER BY `p`.`id`", ['module' => $sModule]);
  1238. }
  1239. public function isPageObject($sObject)
  1240. {
  1241. return (bool)$this->getOne("SELECT 1 FROM `sys_objects_page` WHERE `object` = :object LIMIT 1", ['object' => $sObject]);
  1242. }
  1243. public function isPageUri($sUri)
  1244. {
  1245. return (bool)$this->getOne("SELECT 1 FROM `sys_objects_page` WHERE `uri` = :uri LIMIT 1", ['uri' => $sUri]);
  1246. }
  1247. /**
  1248. * A new page of the module, with copies of the blocks in some cells of another page.
  1249. * The copies are shown in NEO when their source is (active_api: a block NEO has no
  1250. * element for shows as raw data there) and can't be copied again from the builder.
  1251. *
  1252. * @param array $aPage object, uri, title_system, title, module, layout_id, author
  1253. * @param string $sFrom the page whose blocks are copied
  1254. * @param array $aCells cell ids to copy
  1255. * @return int blocks copied, or false when the page wasn't created
  1256. */
  1257. public function insertPageWithBlocks($aPage, $sFrom, $aCells)
  1258. {
  1259. $bInserted = $this->query("INSERT INTO `sys_objects_page` SET `author` = :author, `added` = :added, `object` = :object, `uri` = :uri, `title_system` = :title_system, `title` = :title, `module` = :module, `layout_id` = :layout_id, `visible_for_levels` = 2147483647, `visible_for_levels_editable` = 1, `url` = :url, `meta_robots` = 'noindex', `cache_lifetime` = 0, `cache_editable` = 1, `deletable` = 1", [
  1260. 'author' => (int)$aPage['author'],
  1261. 'added' => time(),
  1262. 'object' => $aPage['object'],
  1263. 'uri' => $aPage['uri'],
  1264. 'title_system' => $aPage['title_system'],
  1265. 'title' => $aPage['title'],
  1266. 'module' => $aPage['module'],
  1267. 'layout_id' => (int)$aPage['layout_id'],
  1268. 'url' => 'page.php?i=' . $aPage['uri'],
  1269. ]);
  1270. if(!$bInserted)
  1271. return false;
  1272. $aBindings = ['object' => $aPage['object'], 'from' => $sFrom];
  1273. $sIn = $this->_getInSql(array_map('intval', $aCells), 'cell', $aBindings);
  1274. return (int)$this->query("INSERT INTO `sys_pages_blocks` (`object`, `cell_id`, `module`, `title_system`, `title`, `designbox_id`, `class`, `visible_for_levels`, `type`, `content`, `deletable`, `copyable`, `active`, `active_api`, `order`)
  1275. SELECT :object, `cell_id`, `module`, `title_system`, `title`, `designbox_id`, `class`, `visible_for_levels`, `type`, `content`, 1, 0, 1, `active_api`, `order`
  1276. FROM `sys_pages_blocks` WHERE `object` = :from AND `active` = 1 AND `cell_id` IN (" . $sIn . ") ORDER BY `cell_id`, `order`", $aBindings);
  1277. }
  1278. /**
  1279. * Whether the classic tracker is injected into the site's pages.
  1280. */
  1281. public function isInjectionActive($sName)
  1282. {
  1283. return (bool)$this->getOne("SELECT `active` FROM `sys_injections` WHERE `name` = :name LIMIT 1", ['name' => $sName]);
  1284. }
  1285. /**
  1286. * Alerts a handler listens to, as rows of unit and action.
  1287. */
  1288. public function getHandlerAlerts($sHandler)
  1289. {
  1290. return $this->getAll("SELECT `a`.`unit`, `a`.`action` FROM `sys_alerts` AS `a` INNER JOIN `sys_alerts_handlers` AS `h` ON `h`.`id` = `a`.`handler_id` WHERE `h`.`name` = :name ORDER BY `a`.`id`", ['name' => $sHandler]);
  1291. }
  1292. protected function _getPageviews($aFilter)
  1293. {
  1294. $aBindings = [];
  1295. $sWhere = $this->_getEventsWhere($aFilter, $aBindings);
  1296. return (int)$this->getOne("SELECT COUNT(*) FROM `" . $this->_oConfig->CNF['TABLE_EVENTS'] . "` AS `e` WHERE " . $sWhere, $aBindings);
  1297. }
  1298. protected function _hasEventDimensions($aFilter)
  1299. {
  1300. foreach($aFilter['dims'] as $sName => $sValue)
  1301. if(isset($this->_aEventDimensions[$sName]))
  1302. return true;
  1303. return false;
  1304. }
  1305. /**
  1306. * A visit's hits are taken to be at most this many seconds before its start or after its
  1307. * last hit, to bound the visits or pageviews a filter across the two tables reads.
  1308. */
  1309. const VISIT_SLACK = 86400;
  1310. /**
  1311. * `$sColumn` (a session id) is a visit that matches session conditions on `sf` and has
  1312. * hits in the period. The visits are found first, once, and the rows only look them up:
  1313. * a correlated EXISTS looked a visit up for every pageview, millions over 90 days, and lets
  1314. * the optimizer group the pageviews in a temporary table. The hint asks MariaDB 12 and
  1315. * MySQL for that order; older servers read it as a comment.
  1316. *
  1317. * @param bool $bCorrelated look each row's visit up instead: for a scan that stops after
  1318. * a few rows (a LIMIT in key order), where finding every visit first costs more
  1319. */
  1320. protected function _getVisitsInSql($sColumn, $aConditions, $aFilter, &$aBindings, $bCorrelated = false)
  1321. {
  1322. if($bCorrelated)
  1323. return "EXISTS (SELECT 1 FROM `" . $this->_oConfig->CNF['TABLE_SESSIONS'] . "` AS `sf` WHERE `sf`.`id` = " . $sColumn . " AND " . implode(' AND ', $aConditions) . ")";
  1324. $aBindings['sf_from'] = (int)$aFilter['from'] - self::VISIT_SLACK;
  1325. $aBindings['sf_to'] = (int)$aFilter['to'] + self::VISIT_SLACK;
  1326. return $sColumn . " IN (SELECT " . self::SQL_SUBQUERY_FIRST . " `sf`.`id` FROM `" . $this->_oConfig->CNF['TABLE_SESSIONS'] . "` AS `sf` WHERE `sf`.`last_seen` >= :sf_from AND `sf`.`started` < :sf_to AND " . implode(' AND ', $aConditions) . ")";
  1327. }
  1328. /**
  1329. * `$sColumn` (a session id) is a visit with a pageview that matches conditions on `ef`,
  1330. * viewed from a day before the period on: the visits come from the `path` key (a page
  1331. * filter) and are looked up once each, see _getVisitsInSql.
  1332. *
  1333. * @param string $sSuffix tells the bindings of several such conditions apart
  1334. */
  1335. protected function _getPageVisitsInSql($sColumn, $aConditions, $aFilter, &$aBindings, $sSuffix = '')
  1336. {
  1337. $aBindings['ef_from' . $sSuffix] = (int)$aFilter['from'] - self::VISIT_SLACK;
  1338. return $sColumn . " IN (SELECT " . self::SQL_SUBQUERY_FIRST . " `ef`.`session_id` FROM `" . $this->_oConfig->CNF['TABLE_EVENTS'] . "` AS `ef` WHERE `ef`.`type` = 'pageview' AND `ef`.`ts` >= :ef_from" . $sSuffix . " AND " . implode(' AND ', $aConditions) . ")";
  1339. }
  1340. /**
  1341. * The sessions table for reports, alias `s`. With a platform filter the optimizer may read
  1342. * the visits of the platform through `platform_seen` (kept for Setup's last hit per
  1343. * platform) instead of the period's range of the covering `started` key; when the kept
  1344. * data is about as long as the period, it did: 7 s instead of 0.4 s for 90 days.
  1345. */
  1346. protected function _getSessionsFrom()
  1347. {
  1348. return "`" . $this->_oConfig->CNF['TABLE_SESSIONS'] . "` AS `s` IGNORE INDEX (`platform_seen`)";
  1349. }
  1350. /**
  1351. * Whether a period is short enough to look the visit of each row up (see _getVisitsInSql):
  1352. * a week of events of one kind are few rows, and finding every visit of the filter first
  1353. * costs a scan of the visits' table (1.5 s at 2.5 million visits).
  1354. */
  1355. protected function _isShortPeriod($aFilter)
  1356. {
  1357. return (int)$aFilter['to'] - (int)$aFilter['from'] <= 8 * 86400;
  1358. }
  1359. /**
  1360. * WHERE for sessions (alias `s`) that started in the period and match the filters. A page
  1361. * filter keeps the sessions that viewed that page.
  1362. */
  1363. protected function _getSessionsWhere($aFilter, &$aBindings)
  1364. {
  1365. $aSql = ['`s`.`started` >= :from', '`s`.`started` < :to'];
  1366. $aBindings['from'] = (int)$aFilter['from'];
  1367. $aBindings['to'] = (int)$aFilter['to'];
  1368. if($aFilter['platform'] !== '') {
  1369. $aSql[] = '`s`.`platform` = :platform';
  1370. $aBindings['platform'] = $aFilter['platform'];
  1371. }
  1372. $aEventSql = [];
  1373. foreach($aFilter['dims'] as $sName => $sValue) {
  1374. if(isset($this->_aSessionDimensions[$sName])) {
  1375. $aSql[] = '`s`.`' . $this->_aSessionDimensions[$sName] . '` = :d_' . $sName;
  1376. $aBindings['d_' . $sName] = $sValue;
  1377. }
  1378. elseif(isset($this->_aEventDimensions[$sName])) {
  1379. $aEventSql[] = '`ef`.`' . $this->_aEventDimensions[$sName] . '` = :d_' . $sName;
  1380. $aBindings['d_' . $sName] = $sValue;
  1381. }
  1382. }
  1383. if(!empty($aEventSql))
  1384. $aSql[] = $this->_getPageVisitsInSql('`s`.`id`', $aEventSql, $aFilter, $aBindings);
  1385. return implode(' AND ', $aSql);
  1386. }
  1387. /**
  1388. * WHERE for pageview events (alias `e`) in the period that match the filters. Session
  1389. * filters keep the events of matching sessions.
  1390. */
  1391. protected function _getEventsWhere($aFilter, &$aBindings, $bCorrelated = false)
  1392. {
  1393. $aSql = ['`e`.`ts` >= :from', '`e`.`ts` < :to', "`e`.`type` = 'pageview'"];
  1394. $aBindings['from'] = (int)$aFilter['from'];
  1395. $aBindings['to'] = (int)$aFilter['to'];
  1396. if($aFilter['platform'] !== '') {
  1397. $aSql[] = '`e`.`platform` = :platform';
  1398. $aBindings['platform'] = $aFilter['platform'];
  1399. }
  1400. $aSessionSql = [];
  1401. foreach($aFilter['dims'] as $sName => $sValue) {
  1402. if(isset($this->_aEventDimensions[$sName])) {
  1403. $aSql[] = '`e`.`' . $this->_aEventDimensions[$sName] . '` = :d_' . $sName;
  1404. $aBindings['d_' . $sName] = $sValue;
  1405. }
  1406. elseif(isset($this->_aSessionDimensions[$sName])) {
  1407. $aSessionSql[] = '`sf`.`' . $this->_aSessionDimensions[$sName] . '` = :d_' . $sName;
  1408. $aBindings['d_' . $sName] = $sValue;
  1409. }
  1410. }
  1411. if(!empty($aSessionSql))
  1412. $aSql[] = $this->_getVisitsInSql('`e`.`session_id`', $aSessionSql, $aFilter, $aBindings, $bCorrelated);
  1413. return implode(' AND ', $aSql);
  1414. }
  1415. /**
  1416. * Bucket start time of a timestamp column. Given a list of starts (months, or days across
  1417. * a daylight saving change), a row goes to the latest start before it. Otherwise hours,
  1418. * days and weeks (from Monday) are fixed lengths shifted by the timezone offset.
  1419. */
  1420. protected function _getBucketSql($sColumn, $sBucket, $iOffset, $aStarts, &$aBindings)
  1421. {
  1422. $iOffset = (int)$iOffset;
  1423. if(!empty($aStarts)) {
  1424. // INTERVAL() finds the last start up to the value by a binary search over the
  1425. // ascending list, ELT() returns that start; a CASE with a branch per start costs a
  1426. // comparison per bucket for every row (90 at 90 days). Before the first start: 0.
  1427. $aFind = [];
  1428. $aPick = [];
  1429. foreach(array_values($aStarts) as $i => $iStart) {
  1430. $aFind[] = ':b' . $i;
  1431. $aPick[] = ':b' . $i . 'v';
  1432. $aBindings['b' . $i] = (int)$iStart;
  1433. $aBindings['b' . $i . 'v'] = (int)$iStart;
  1434. }
  1435. return 'IFNULL(ELT(INTERVAL(' . $sColumn . ', ' . implode(', ', $aFind) . '), ' . implode(', ', $aPick) . '), 0)';
  1436. }
  1437. switch($sBucket) {
  1438. case 'hour':
  1439. return 'FLOOR((' . $sColumn . ' + ' . $iOffset . ') / 3600) * 3600 - ' . $iOffset;
  1440. case 'week':
  1441. // 1970-01-05 was a Monday
  1442. return 'FLOOR((' . $sColumn . ' + ' . $iOffset . ' - 345600) / 604800) * 604800 + 345600 - ' . $iOffset;
  1443. default:
  1444. return 'FLOOR((' . $sColumn . ' + ' . $iOffset . ') / 86400) * 86400 - ' . $iOffset;
  1445. }
  1446. }
  1447. /*
  1448. * Health: errors and vitals
  1449. */
  1450. /**
  1451. * Columns of an error hit that filters read directly; other dimensions go through its visit.
  1452. */
  1453. protected $_aErrorHitDimensions = ['app_version', 'browser', 'os', 'device', 'country', 'path'];
  1454. /**
  1455. * Columns an error breakdown may group by.
  1456. */
  1457. protected $_aErrorBreakdowns = ['platform', 'app_version', 'browser', 'os', 'device', 'country', 'route'];
  1458. /**
  1459. * Raise a visit's error count by `$iCount` unless that passes `$iMax`.
  1460. *
  1461. * @return bool whether the errors fit
  1462. */
  1463. public function addSessionErrors($iSessionId, $iCount, $iMax)
  1464. {
  1465. return (int)$this->query("UPDATE `" . $this->_oConfig->CNF['TABLE_SESSIONS'] . "` SET `errors` = `errors` + :n WHERE `id` = :id AND `errors` + :n2 <= :max", [
  1466. 'id' => (int)$iSessionId,
  1467. 'n' => (int)$iCount,
  1468. 'n2' => (int)$iCount,
  1469. 'max' => (int)$iMax,
  1470. ]) > 0;
  1471. }
  1472. public function getErrorGroupByFingerprint($sFingerprint)
  1473. {
  1474. return $this->getRow("SELECT `id`, `status`, `status_changed` FROM `" . $this->_oConfig->CNF['TABLE_ERRORS'] . "` WHERE `fingerprint` = :fingerprint", ['fingerprint' => $sFingerprint]);
  1475. }
  1476. /**
  1477. * Error groups created in a time span (server time, whatever the age of their first error).
  1478. */
  1479. public function getErrorGroupsCreated($iFrom, $iTo)
  1480. {
  1481. return (int)$this->getOne("SELECT COUNT(*) FROM `" . $this->_oConfig->CNF['TABLE_ERRORS'] . "` WHERE `created` >= :from AND `created` <= :to", ['from' => (int)$iFrom, 'to' => (int)$iTo]);
  1482. }
  1483. /**
  1484. * Add an occurrence to an error's group, creating the group with its sample stack when it
  1485. * is new. The stack is replaced by a newer one once a day, so it follows new releases.
  1486. * The message and the source are the latest occurrence's, so no visitor text in them
  1487. * outlives the group's last occurrence (pruning clears old stacks, see
  1488. * clearErrorStacksBefore). MySQL assigns left to right: `stack` comes before `stack_ts`,
  1489. * the message and the source before `last_seen`.
  1490. *
  1491. * @param string $sFingerprint 16 bytes, see BxAnalyticsErrors::getFingerprint
  1492. * @param array $aError kind, message, source, line, col, status, stack, fatal, repeats
  1493. * @param int $iTime when it happened
  1494. * @param int $iNow server time
  1495. * @param int $iId the group's id when it is known to exist: an update, which doesn't use
  1496. * up an auto-increment id as INSERT ... ON DUPLICATE KEY UPDATE does
  1497. * @return int the group's id, 0 on failure
  1498. */
  1499. public function saveErrorGroup($sFingerprint, $aError, $iTime, $iNow, $iId = 0)
  1500. {
  1501. if($iId) {
  1502. $bResult = $this->query("UPDATE `" . $this->_oConfig->CNF['TABLE_ERRORS'] . "` SET
  1503. `message` = IF(:message_ts >= `last_seen`, :message, `message`),
  1504. `source` = IF(:source_ts >= `last_seen`, :source, `source`),
  1505. `first_seen` = LEAST(`first_seen`, :first_seen),
  1506. `last_seen` = GREATEST(`last_seen`, :last_seen),
  1507. `occurrences` = `occurrences` + :occurrences,
  1508. `fatal` = GREATEST(`fatal`, :fatal),
  1509. `stack` = IF(:stack != '' AND `stack_ts` < :stack_ts - 86400, :stack_new, `stack`),
  1510. `stack_ts` = IF(:stack_check != '' AND `stack_ts` < :stack_ts_check - 86400, :stack_ts_new, `stack_ts`)
  1511. WHERE `id` = :id", [
  1512. 'id' => (int)$iId,
  1513. 'message' => $aError['message'],
  1514. 'message_ts' => (int)$iTime,
  1515. 'source' => $aError['source'],
  1516. 'source_ts' => (int)$iTime,
  1517. 'first_seen' => (int)$iTime,
  1518. 'last_seen' => (int)$iTime,
  1519. 'occurrences' => isset($aError['repeats']) ? (int)$aError['repeats'] : 1,
  1520. 'fatal' => (int)$aError['fatal'],
  1521. 'stack' => $aError['stack'],
  1522. 'stack_new' => $aError['stack'],
  1523. 'stack_check' => $aError['stack'],
  1524. 'stack_ts' => (int)$iNow,
  1525. 'stack_ts_check' => (int)$iNow,
  1526. 'stack_ts_new' => (int)$iNow,
  1527. ]);
  1528. return $bResult !== false ? (int)$iId : 0;
  1529. }
  1530. $bResult = $this->query("INSERT INTO `" . $this->_oConfig->CNF['TABLE_ERRORS'] . "` (`fingerprint`, `kind`, `message`, `source`, `line`, `col`, `status_code`, `stack`, `stack_ts`, `first_seen`, `last_seen`, `occurrences`, `fatal`, `created`) VALUES (:fingerprint, :kind, :message, :source, :line, :col, :status_code, :stack, :stack_ts, :first_seen, :last_seen, :occurrences, :fatal, :created)
  1531. ON DUPLICATE KEY UPDATE `id` = LAST_INSERT_ID(`id`),
  1532. `message` = IF(VALUES(`last_seen`) >= `last_seen`, VALUES(`message`), `message`),
  1533. `source` = IF(VALUES(`last_seen`) >= `last_seen`, VALUES(`source`), `source`),
  1534. `first_seen` = LEAST(`first_seen`, VALUES(`first_seen`)),
  1535. `last_seen` = GREATEST(`last_seen`, VALUES(`last_seen`)),
  1536. `occurrences` = `occurrences` + VALUES(`occurrences`),
  1537. `fatal` = GREATEST(`fatal`, VALUES(`fatal`)),
  1538. `stack` = IF(VALUES(`stack`) != '' AND `stack_ts` < VALUES(`stack_ts`) - 86400, VALUES(`stack`), `stack`),
  1539. `stack_ts` = IF(VALUES(`stack`) != '' AND `stack_ts` < VALUES(`stack_ts`) - 86400, VALUES(`stack_ts`), `stack_ts`)", [
  1540. 'fingerprint' => $sFingerprint,
  1541. 'kind' => $aError['kind'],
  1542. 'message' => $aError['message'],
  1543. 'source' => $aError['source'],
  1544. 'line' => (int)$aError['line'],
  1545. 'col' => (int)$aError['col'],
  1546. 'status_code' => (int)$aError['status'],
  1547. 'stack' => $aError['stack'],
  1548. 'stack_ts' => $aError['stack'] !== '' ? (int)$iNow : 0,
  1549. 'first_seen' => (int)$iTime,
  1550. 'last_seen' => (int)$iTime,
  1551. 'occurrences' => isset($aError['repeats']) ? (int)$aError['repeats'] : 1,
  1552. 'fatal' => (int)$aError['fatal'],
  1553. 'created' => (int)$iNow,
  1554. ]);
  1555. return $bResult !== false ? (int)$this->lastId() : 0;
  1556. }
  1557. /**
  1558. * Set an error group's status (open, resolved, ignored), optionally only from another one.
  1559. */
  1560. public function setErrorStatus($iId, $sStatus, $iTime, $sFrom = '')
  1561. {
  1562. $aBindings = ['id' => (int)$iId, 'status' => $sStatus, 'time' => (int)$iTime];
  1563. $sWhere = '';
  1564. if($sFrom !== '') {
  1565. $sWhere = ' AND `status` = :from';
  1566. $aBindings['from'] = $sFrom;
  1567. }
  1568. return $this->query("UPDATE `" . $this->_oConfig->CNF['TABLE_ERRORS'] . "` SET `status` = :status, `status_changed` = :time WHERE `id` = :id" . $sWhere, $aBindings) !== false;
  1569. }
  1570. public function getErrorGroup($iId)
  1571. {
  1572. return $this->getRow("SELECT `id`, `kind`, `message`, `source`, `line`, `col`, `status_code`, `stack`, `first_seen`, `last_seen`, `occurrences`, `fatal`, `status`, `status_changed` FROM `" . $this->_oConfig->CNF['TABLE_ERRORS'] . "` WHERE `id` = :id", ['id' => (int)$iId]);
  1573. }
  1574. public function getErrorGroups($aIds)
  1575. {
  1576. if(empty($aIds))
  1577. return [];
  1578. $aBindings = [];
  1579. $aIn = [];
  1580. foreach(array_values($aIds) as $i => $iId) {
  1581. $aIn[] = ':id' . $i;
  1582. $aBindings['id' . $i] = (int)$iId;
  1583. }
  1584. return $this->getAllWithKey("SELECT `id`, `kind`, `message`, `source`, `line`, `col`, `status_code`, `first_seen`, `last_seen`, `occurrences`, `fatal`, `status` FROM `" . $this->_oConfig->CNF['TABLE_ERRORS'] . "` WHERE `id` IN (" . implode(', ', $aIn) . ")", 'id', $aBindings);
  1585. }
  1586. /**
  1587. * Insert error hits in one query.
  1588. *
  1589. * @return int rows inserted
  1590. */
  1591. public function insertErrorHits($aRows)
  1592. {
  1593. return $this->_insertRows($this->_oConfig->CNF['TABLE_ERROR_HITS'], $aRows);
  1594. }
  1595. /**
  1596. * One page view's vitals; metrics not measured are left NULL.
  1597. */
  1598. public function insertVitals($aRow)
  1599. {
  1600. return $this->_insertRows($this->_oConfig->CNF['TABLE_VITALS'], [$aRow]);
  1601. }
  1602. /**
  1603. * Error hits, visits with errors, visitors, and visits that ended in a fatal error on some
  1604. * platforms, for error groups in some statuses.
  1605. *
  1606. * @param array $aFilter report filter
  1607. * @param array $aStatuses group statuses
  1608. * @param array $aFatalPlatforms platforms whose fatal errors are crashes (the apps)
  1609. */
  1610. public function getErrorTotals($aFilter, $aStatuses, $aFatalPlatforms = [])
  1611. {
  1612. $aBindings = [];
  1613. $sWhere = $this->_getErrorHitsWhere($aFilter, $aStatuses, $aBindings);
  1614. // a crash outside any visit known (an app that stopped before its first page, or whose
  1615. // visit could not be found when it reported the crash on its next launch) is a visit too
  1616. $sFatal = '0';
  1617. if(!empty($aFatalPlatforms))
  1618. $sFatal = "COUNT(DISTINCT CASE WHEN `h`.`fatal` = 1 AND `h`.`session_id` > 0 AND `h`.`platform` IN (" . $this->_getInSql($aFatalPlatforms, 'fp', $aBindings) . ") THEN `h`.`session_id` END) + COUNT(CASE WHEN `h`.`fatal` = 1 AND `h`.`session_id` = 0 AND `h`.`platform` IN (" . $this->_getInSql($aFatalPlatforms, 'fu', $aBindings) . ") THEN `h`.`id` END)";
  1619. return $this->getRow("SELECT IFNULL(SUM(`h`.`repeats`), 0) AS `hits`, COUNT(DISTINCT NULLIF(`h`.`session_id`, 0)) AS `sessions`, COUNT(DISTINCT `h`.`visitor`) AS `users`, " . $sFatal . " AS `fatal_sessions` FROM `" . $this->_oConfig->CNF['TABLE_ERROR_HITS'] . "` AS `h` INNER JOIN `" . $this->_oConfig->CNF['TABLE_ERRORS'] . "` AS `g` ON `g`.`id` = `h`.`error_id` WHERE " . $sWhere, $aBindings);
  1620. }
  1621. /**
  1622. * Visits that started in the period per platform.
  1623. */
  1624. public function getSessionsPerPlatform($aFilter)
  1625. {
  1626. $aBindings = [];
  1627. $sWhere = $this->_getSessionsWhere($aFilter, $aBindings);
  1628. return $this->getPairs("SELECT `s`.`platform`, COUNT(*) AS `sessions` FROM " . $this->_getSessionsFrom() . " WHERE " . $sWhere . " GROUP BY `s`.`platform`", 'platform', 'sessions', $aBindings);
  1629. }
  1630. /**
  1631. * Error groups with their hits in the period, most frequent first.
  1632. *
  1633. * @param int $iFirstSeenFrom only groups first seen since this time (new errors), 0 for any
  1634. */
  1635. public function getErrorList($aFilter, $aStatuses, $iFirstSeenFrom, $iLimit, $iStart = 0)
  1636. {
  1637. $aBindings = [];
  1638. $sWhere = $this->_getErrorHitsWhere($aFilter, $aStatuses, $aBindings, $iFirstSeenFrom);
  1639. return $this->getAll("SELECT `h`.`error_id`, SUM(`h`.`repeats`) AS `count`, COUNT(DISTINCT NULLIF(`h`.`session_id`, 0)) AS `sessions`, COUNT(DISTINCT `h`.`visitor`) AS `users`, GROUP_CONCAT(DISTINCT `h`.`platform` ORDER BY `h`.`platform`) AS `platforms`, MAX(`h`.`fatal`) AS `fatal`, MAX(`h`.`ts`) AS `last_in` FROM `" . $this->_oConfig->CNF['TABLE_ERROR_HITS'] . "` AS `h` INNER JOIN `" . $this->_oConfig->CNF['TABLE_ERRORS'] . "` AS `g` ON `g`.`id` = `h`.`error_id` WHERE " . $sWhere . " GROUP BY `h`.`error_id` ORDER BY `count` DESC, `h`.`error_id` DESC LIMIT " . max(0, (int)$iStart) . ", " . max(1, min(1000, (int)$iLimit)), $aBindings);
  1640. }
  1641. public function getErrorListCount($aFilter, $aStatuses, $iFirstSeenFrom = 0)
  1642. {
  1643. $aBindings = [];
  1644. $sWhere = $this->_getErrorHitsWhere($aFilter, $aStatuses, $aBindings, $iFirstSeenFrom);
  1645. return (int)$this->getOne("SELECT COUNT(DISTINCT `h`.`error_id`) FROM `" . $this->_oConfig->CNF['TABLE_ERROR_HITS'] . "` AS `h` INNER JOIN `" . $this->_oConfig->CNF['TABLE_ERRORS'] . "` AS `g` ON `g`.`id` = `h`.`error_id` WHERE " . $sWhere, $aBindings);
  1646. }
  1647. /**
  1648. * Hits per time bucket of some error groups, or of one with visitors too.
  1649. *
  1650. * @return array [error id => [bucket => count]]
  1651. */
  1652. public function getErrorTrends($aFilter, $aIds, $sBucket, $iOffset, $aStarts = [])
  1653. {
  1654. if(empty($aIds))
  1655. return [];
  1656. $aBindings = [];
  1657. $sWhere = $this->_getErrorHitsWhere($aFilter, [], $aBindings) . " AND `h`.`error_id` IN (" . $this->_getInSql($aIds, 'e', $aBindings) . ")";
  1658. $sBucketSql = $this->_getBucketSql('`h`.`ts`', $sBucket, $iOffset, $aStarts, $aBindings);
  1659. $aResult = [];
  1660. foreach($this->getAll("SELECT `h`.`error_id`, " . $sBucketSql . " AS `bucket`, SUM(`h`.`repeats`) AS `count` FROM `" . $this->_oConfig->CNF['TABLE_ERROR_HITS'] . "` AS `h` WHERE " . $sWhere . " GROUP BY `h`.`error_id`, `bucket`", $aBindings) as $aRow)
  1661. $aResult[(int)$aRow['error_id']][(int)$aRow['bucket']] = (int)$aRow['count'];
  1662. return $aResult;
  1663. }
  1664. public function getErrorSeries($aFilter, $iId, $sBucket, $iOffset, $aStarts = [])
  1665. {
  1666. $aBindings = ['error' => (int)$iId];
  1667. $sWhere = $this->_getErrorHitsWhere($aFilter, [], $aBindings) . " AND `h`.`error_id` = :error";
  1668. $sBucketSql = $this->_getBucketSql('`h`.`ts`', $sBucket, $iOffset, $aStarts, $aBindings);
  1669. return $this->getAllWithKey("SELECT " . $sBucketSql . " AS `bucket`, SUM(`h`.`repeats`) AS `count`, COUNT(DISTINCT `h`.`visitor`) AS `users` FROM `" . $this->_oConfig->CNF['TABLE_ERROR_HITS'] . "` AS `h` WHERE " . $sWhere . " GROUP BY `bucket` ORDER BY `bucket`", 'bucket', $aBindings);
  1670. }
  1671. /**
  1672. * One error group's hits, visits and visitors in the period.
  1673. */
  1674. public function getErrorStats($aFilter, $iId)
  1675. {
  1676. $aBindings = ['error' => (int)$iId];
  1677. $sWhere = $this->_getErrorHitsWhere($aFilter, [], $aBindings) . " AND `h`.`error_id` = :error";
  1678. return $this->getRow("SELECT IFNULL(SUM(`h`.`repeats`), 0) AS `count`, COUNT(DISTINCT NULLIF(`h`.`session_id`, 0)) AS `sessions`, COUNT(DISTINCT `h`.`visitor`) AS `users` FROM `" . $this->_oConfig->CNF['TABLE_ERROR_HITS'] . "` AS `h` WHERE " . $sWhere, $aBindings);
  1679. }
  1680. /**
  1681. * One error group's hits by a client column: platform, app version, browser, OS, device,
  1682. * country or route.
  1683. */
  1684. public function getErrorBreakdown($aFilter, $iId, $sColumn, $iLimit = 10)
  1685. {
  1686. if(!in_array($sColumn, $this->_aErrorBreakdowns, true))
  1687. return [];
  1688. $aBindings = ['error' => (int)$iId];
  1689. $sWhere = $this->_getErrorHitsWhere($aFilter, [], $aBindings) . " AND `h`.`error_id` = :error";
  1690. return $this->getAll("SELECT `h`.`" . $sColumn . "` AS `value`, SUM(`h`.`repeats`) AS `count`, COUNT(DISTINCT `h`.`visitor`) AS `users` FROM `" . $this->_oConfig->CNF['TABLE_ERROR_HITS'] . "` AS `h` WHERE " . $sWhere . " GROUP BY `h`.`" . $sColumn . "` ORDER BY `count` DESC LIMIT " . max(1, min(100, (int)$iLimit)), $aBindings);
  1691. }
  1692. /**
  1693. * One error group's latest hits in the period.
  1694. */
  1695. public function getErrorOccurrences($aFilter, $iId, $iLimit = 20)
  1696. {
  1697. $aBindings = ['error' => (int)$iId];
  1698. $sWhere = $this->_getErrorHitsWhere($aFilter, [], $aBindings) . " AND `h`.`error_id` = :error";
  1699. return $this->getAll("SELECT `h`.`ts`, `h`.`session_id`, `h`.`platform`, `h`.`app_version`, `h`.`browser`, `h`.`os`, `h`.`device`, `h`.`country`, `h`.`path`, `h`.`repeats`, `h`.`fatal` FROM `" . $this->_oConfig->CNF['TABLE_ERROR_HITS'] . "` AS `h` WHERE " . $sWhere . " ORDER BY `h`.`ts` DESC, `h`.`id` DESC LIMIT " . max(1, min(100, (int)$iLimit)), $aBindings);
  1700. }
  1701. /**
  1702. * A log-scale histogram of a metric (see BxAnalyticsVitals::getBucket), optionally per
  1703. * platform, page (route or path) or time bucket.
  1704. *
  1705. * @param array $aFilter report filter
  1706. * @param string $sMetric a BxAnalyticsVitals::METRICS column
  1707. * @param string $sDevice all, mobile (with tablets) or desktop
  1708. * @param string $sGroup '', platform, route, path or bucket
  1709. * @param array $aIn values of the group to keep (pages)
  1710. * @param array $aBucket [bucket, offset, starts] when grouped by time
  1711. * @return array rows of grp, hb, side (0 up to the good threshold, 1 between, 2 above
  1712. * the poor one), n
  1713. */
  1714. public function getVitalsHistogram($aFilter, $sMetric, $sDevice, $sGroup = '', $aIn = [], $aBucket = [])
  1715. {
  1716. if(!in_array($sMetric, BxAnalyticsVitals::METRICS, true))
  1717. return [];
  1718. $aBindings = [
  1719. 'side_good' => (int)$this->_oConfig->CNF['VITALS'][$sMetric][0],
  1720. 'side_poor' => (int)$this->_oConfig->CNF['VITALS'][$sMetric][1],
  1721. ];
  1722. $sWhere = $this->_getVitalsWhere($aFilter, $sDevice, $aBindings) . " AND `v`.`" . $sMetric . "` IS NOT NULL";
  1723. switch($sGroup) {
  1724. case 'platform':
  1725. case 'route':
  1726. case 'path':
  1727. $sGroupSql = "`v`.`" . $sGroup . "`";
  1728. if(!empty($aIn))
  1729. $sWhere .= " AND `v`.`" . $sGroup . "` IN (" . $this->_getInSql($aIn, 'g', $aBindings) . ")";
  1730. break;
  1731. case 'bucket':
  1732. $sGroupSql = $this->_getBucketSql('`v`.`ts`', $aBucket[0], $aBucket[1], $aBucket[2], $aBindings);
  1733. break;
  1734. default:
  1735. $sGroupSql = "''";
  1736. }
  1737. return $this->getAll("SELECT " . $sGroupSql . " AS `grp`, " . $this->_getVitalsBucketSql($sMetric) . " AS `hb`, (`v`.`" . $sMetric . "` > :side_good) + (`v`.`" . $sMetric . "` > :side_poor) AS `side`, COUNT(*) AS `n` FROM `" . $this->_oConfig->CNF['TABLE_VITALS'] . "` AS `v` WHERE " . $sWhere . " GROUP BY `grp`, `hb`, `side`", $aBindings);
  1738. }
  1739. /**
  1740. * Log-scale histograms of several metrics in one pass over the period: each row is
  1741. * paired with every metric (a derived table of their names), and the metric's own
  1742. * column is bucketed.
  1743. *
  1744. * @return array rows of grp (the metric), hb, n
  1745. */
  1746. public function getVitalsHistograms($aFilter, $aMetrics, $sDevice)
  1747. {
  1748. $aMetrics = array_values(array_intersect(BxAnalyticsVitals::METRICS, $aMetrics));
  1749. if(empty($aMetrics))
  1750. return [];
  1751. $aBindings = [];
  1752. $sWhere = $this->_getVitalsWhere($aFilter, $sDevice, $aBindings);
  1753. $aNames = [];
  1754. $aValue = [];
  1755. $aBucket = [];
  1756. foreach($aMetrics as $sMetric) {
  1757. // names from BxAnalyticsVitals::METRICS only
  1758. $aNames[] = "SELECT '" . $sMetric . "' AS `metric`";
  1759. $aValue[] = "WHEN '" . $sMetric . "' THEN `v`.`" . $sMetric . "`";
  1760. $aBucket[] = "WHEN '" . $sMetric . "' THEN " . $this->_getVitalsBucketSql($sMetric);
  1761. }
  1762. // the vitals table is read first and once; the metric names join to each row
  1763. return $this->getAll("SELECT `m`.`metric` AS `grp`, CASE `m`.`metric` " . implode(' ', $aBucket) . " END AS `hb`, COUNT(*) AS `n` FROM `" . $this->_oConfig->CNF['TABLE_VITALS'] . "` AS `v` STRAIGHT_JOIN (" . implode(' UNION ALL ', $aNames) . ") AS `m` WHERE " . $sWhere . " AND CASE `m`.`metric` " . implode(' ', $aValue) . " END IS NOT NULL GROUP BY `grp`, `hb`", $aBindings);
  1764. }
  1765. /**
  1766. * A metric's histogram bucket, see BxAnalyticsVitals::getBucket.
  1767. */
  1768. protected function _getVitalsBucketSql($sMetric)
  1769. {
  1770. return "IF(`v`.`" . $sMetric . "` = 0, -1, FLOOR(LN(`v`.`" . $sMetric . "`) * " . BxAnalyticsVitals::BUCKETS . "))";
  1771. }
  1772. /**
  1773. * Samples, good and poor values of every metric in one pass.
  1774. *
  1775. * @param array $aThresholds metric => [good up to, poor above], stored units
  1776. * @return array metric => [samples, good, poor]
  1777. */
  1778. public function getVitalsDistribution($aFilter, $sDevice, $aThresholds)
  1779. {
  1780. $aBindings = [];
  1781. $sWhere = $this->_getVitalsWhere($aFilter, $sDevice, $aBindings);
  1782. $aSql = [];
  1783. foreach(BxAnalyticsVitals::METRICS as $sMetric) {
  1784. $aSql[] = "SUM(`v`.`" . $sMetric . "` IS NOT NULL) AS `" . $sMetric . "_n`, SUM(`v`.`" . $sMetric . "` <= :" . $sMetric . "_good) AS `" . $sMetric . "_good`, SUM(`v`.`" . $sMetric . "` > :" . $sMetric . "_poor) AS `" . $sMetric . "_poor`";
  1785. $aBindings[$sMetric . '_good'] = (int)$aThresholds[$sMetric][0];
  1786. $aBindings[$sMetric . '_poor'] = (int)$aThresholds[$sMetric][1];
  1787. }
  1788. $aRow = $this->getRow("SELECT " . implode(', ', $aSql) . " FROM `" . $this->_oConfig->CNF['TABLE_VITALS'] . "` AS `v` WHERE " . $sWhere, $aBindings);
  1789. $aResult = [];
  1790. foreach(BxAnalyticsVitals::METRICS as $sMetric)
  1791. $aResult[$sMetric] = [
  1792. 'samples' => isset($aRow[$sMetric . '_n']) ? (int)$aRow[$sMetric . '_n'] : 0,
  1793. 'good' => isset($aRow[$sMetric . '_good']) ? (int)$aRow[$sMetric . '_good'] : 0,
  1794. 'poor' => isset($aRow[$sMetric . '_poor']) ? (int)$aRow[$sMetric . '_poor'] : 0,
  1795. ];
  1796. return $aResult;
  1797. }
  1798. /**
  1799. * Pages (routes or paths) with the most samples of a metric.
  1800. */
  1801. public function getVitalsPages($aFilter, $sMetric, $sDevice, $sGroup, $iLimit, $iStart = 0)
  1802. {
  1803. if(!in_array($sMetric, BxAnalyticsVitals::METRICS, true) || !in_array($sGroup, ['route', 'path'], true))
  1804. return [];
  1805. $aBindings = [];
  1806. $sWhere = $this->_getVitalsWhere($aFilter, $sDevice, $aBindings) . " AND `v`.`" . $sMetric . "` IS NOT NULL";
  1807. return $this->getPairs("SELECT `v`.`" . $sGroup . "` AS `value`, COUNT(*) AS `n` FROM `" . $this->_oConfig->CNF['TABLE_VITALS'] . "` AS `v` WHERE " . $sWhere . " GROUP BY `v`.`" . $sGroup . "` ORDER BY `n` DESC, `value` LIMIT " . max(0, (int)$iStart) . ", " . max(1, min(1000, (int)$iLimit)), 'value', 'n', $aBindings);
  1808. }
  1809. public function getVitalsPagesCount($aFilter, $sMetric, $sDevice, $sGroup)
  1810. {
  1811. if(!in_array($sMetric, BxAnalyticsVitals::METRICS, true) || !in_array($sGroup, ['route', 'path'], true))
  1812. return 0;
  1813. $aBindings = [];
  1814. $sWhere = $this->_getVitalsWhere($aFilter, $sDevice, $aBindings) . " AND `v`.`" . $sMetric . "` IS NOT NULL";
  1815. return (int)$this->getOne("SELECT COUNT(DISTINCT `v`.`" . $sGroup . "`) FROM `" . $this->_oConfig->CNF['TABLE_VITALS'] . "` AS `v` WHERE " . $sWhere, $aBindings);
  1816. }
  1817. /**
  1818. * WHERE for error hits (alias `h`, joined to their group `g` when statuses are given) in
  1819. * the period. Filters use the hit's own client columns; other dimensions go through its
  1820. * visit, so errors outside a visit drop out of those.
  1821. */
  1822. protected function _getErrorHitsWhere($aFilter, $aStatuses, &$aBindings, $iFirstSeenFrom = 0)
  1823. {
  1824. $sWhere = $this->_getHealthWhere('h', $this->_aErrorHitDimensions, $aFilter, $aBindings);
  1825. if(!empty($aStatuses))
  1826. $sWhere .= " AND `g`.`status` IN (" . $this->_getInSql($aStatuses, 'st', $aBindings) . ")";
  1827. if($iFirstSeenFrom) {
  1828. $sWhere .= " AND `g`.`first_seen` >= :first_seen";
  1829. $aBindings['first_seen'] = (int)$iFirstSeenFrom;
  1830. }
  1831. return $sWhere;
  1832. }
  1833. /**
  1834. * WHERE for vitals (alias `v`) in the period, on a kind of device.
  1835. */
  1836. protected function _getVitalsWhere($aFilter, $sDevice, &$aBindings)
  1837. {
  1838. $sWhere = $this->_getHealthWhere('v', ['device', 'path'], $aFilter, $aBindings);
  1839. if($sDevice == 'mobile')
  1840. $sWhere .= " AND `v`.`device` IN ('mobile', 'tablet')";
  1841. elseif($sDevice == 'desktop')
  1842. $sWhere .= " AND `v`.`device` = 'desktop'";
  1843. return $sWhere;
  1844. }
  1845. /**
  1846. * Period, platform and dimension filters for a health table with `ts`, `platform` and
  1847. * `session_id` columns. Dimensions in `$aOwn` are columns of the table.
  1848. */
  1849. protected function _getHealthWhere($sAlias, $aOwn, $aFilter, &$aBindings)
  1850. {
  1851. $aSql = ['`' . $sAlias . '`.`ts` >= :from', '`' . $sAlias . '`.`ts` < :to'];
  1852. $aBindings['from'] = (int)$aFilter['from'];
  1853. $aBindings['to'] = (int)$aFilter['to'];
  1854. if($aFilter['platform'] !== '') {
  1855. $aSql[] = '`' . $sAlias . '`.`platform` = :platform';
  1856. $aBindings['platform'] = $aFilter['platform'];
  1857. }
  1858. $aThrough = [];
  1859. foreach($aFilter['dims'] as $sName => $sValue) {
  1860. if(in_array($sName, $aOwn, true)) {
  1861. $aSql[] = '`' . $sAlias . '`.`' . $sName . '` = :d_' . $sName;
  1862. // health hits keep only the page parameters of a path, a pageview's path its whole query
  1863. $aBindings['d_' . $sName] = $sName == 'path' ? get_mb_substr(BxAnalyticsErrors::getCleanPath($sValue), 0, 255) : $sValue;
  1864. }
  1865. elseif(isset($this->_aSessionDimensions[$sName])) {
  1866. $aThrough[] = '`sf`.`' . $this->_aSessionDimensions[$sName] . '` = :d_' . $sName;
  1867. $aBindings['d_' . $sName] = $sValue;
  1868. }
  1869. elseif(isset($this->_aEventDimensions[$sName])) {
  1870. $aSql[] = $this->_getPageVisitsInSql('`' . $sAlias . '`.`session_id`', ['`ef`.`' . $this->_aEventDimensions[$sName] . '` = :d_' . $sName], $aFilter, $aBindings, '_' . $sName);
  1871. $aBindings['d_' . $sName] = $sValue;
  1872. }
  1873. }
  1874. // errors and vitals are few rows next to pageviews: each looks its visit up
  1875. if(!empty($aThrough))
  1876. $aSql[] = $this->_getVisitsInSql('`' . $sAlias . '`.`session_id`', $aThrough, $aFilter, $aBindings, true);
  1877. return implode(' AND ', $aSql);
  1878. }
  1879. /**
  1880. * Placeholders for an IN list, with their bindings.
  1881. */
  1882. protected function _getInSql($aValues, $sPrefix, &$aBindings)
  1883. {
  1884. $aIn = [];
  1885. foreach(array_values($aValues) as $i => $mixedValue) {
  1886. $aIn[] = ':' . $sPrefix . $i;
  1887. $aBindings[$sPrefix . $i] = $mixedValue;
  1888. }
  1889. return implode(', ', $aIn);
  1890. }
  1891. /**
  1892. * Insert rows with the same keys in one query.
  1893. */
  1894. protected function _insertRows($sTable, $aRows)
  1895. {
  1896. if(empty($aRows))
  1897. return 0;
  1898. $aFields = array_keys(reset($aRows));
  1899. $aBindings = [];
  1900. $aValues = [];
  1901. foreach(array_values($aRows) as $i => $aRow) {
  1902. $aPlaceholders = [];
  1903. foreach($aFields as $sField) {
  1904. $aPlaceholders[] = ':' . $sField . '_' . $i;
  1905. $aBindings[$sField . '_' . $i] = $aRow[$sField];
  1906. }
  1907. $aValues[] = '(' . implode(', ', $aPlaceholders) . ')';
  1908. }
  1909. return (int)$this->query("INSERT INTO `" . $sTable . "` (`" . implode('`, `', $aFields) . "`) VALUES " . implode(', ', $aValues), $aBindings);
  1910. }
  1911. /*
  1912. * Revenue: the ledger and subscriptions
  1913. */
  1914. /**
  1915. * Ledger columns a revenue split may group by, as name => column.
  1916. */
  1917. protected $_aRevenueDimensions = [
  1918. 'module' => 'module',
  1919. 'provider' => 'provider',
  1920. 'platform' => 'platform',
  1921. 'first_channel' => 'first_channel',
  1922. 'first_referrer_host' => 'first_referrer_host',
  1923. 'first_utm_campaign' => 'first_utm_campaign',
  1924. 'last_channel' => 'last_channel',
  1925. 'last_referrer_host' => 'last_referrer_host',
  1926. 'last_utm_campaign' => 'last_utm_campaign',
  1927. 'seller_id' => 'seller_id',
  1928. ];
  1929. /**
  1930. * Add a money movement unless the same movement (payment module, transaction, kind) is
  1931. * there already; returns 1 when it was added.
  1932. */
  1933. public function insertRevenue($aRow)
  1934. {
  1935. $aBindings = [];
  1936. foreach($aRow as $sField => $mixedValue)
  1937. $aBindings[$sField] = $mixedValue;
  1938. $sFields = '`' . implode('`, `', array_keys($aBindings)) . '`';
  1939. $sValues = ':' . implode(', :', array_keys($aBindings));
  1940. return (int)$this->query("INSERT IGNORE INTO `" . $this->_oConfig->CNF['TABLE_REVENUE'] . "` (" . $sFields . ") VALUES (" . $sValues . ")", $aBindings);
  1941. }
  1942. /**
  1943. * Ledger rows of an order, of some kinds, in transaction order.
  1944. */
  1945. public function getRevenueRows($sPayment, $iPendingId, $aKinds = [])
  1946. {
  1947. $aBindings = ['payment' => $sPayment, 'pending' => (int)$iPendingId];
  1948. return $this->getAll("SELECT * FROM `" . $this->_oConfig->CNF['TABLE_REVENUE'] . "` WHERE `payment` = :payment AND `pending_id` = :pending" . $this->_getKindsSql($aKinds, $aBindings) . " ORDER BY `transaction_id`, `id`", $aBindings);
  1949. }
  1950. /**
  1951. * The ledger row of a transaction, of some kinds.
  1952. */
  1953. public function getRevenueRow($sPayment, $iTransactionId, $aKinds = [])
  1954. {
  1955. $aBindings = ['payment' => $sPayment, 'transaction' => (int)$iTransactionId];
  1956. return $this->getRow("SELECT * FROM `" . $this->_oConfig->CNF['TABLE_REVENUE'] . "` WHERE `payment` = :payment AND `transaction_id` = :transaction" . $this->_getKindsSql($aKinds, $aBindings) . " ORDER BY `id` LIMIT 1", $aBindings);
  1957. }
  1958. /**
  1959. * Time of the first paid charge of each order the ledger has (a trial's registration is
  1960. * not one): [pending id => time].
  1961. */
  1962. public function getRevenueFirstCharges($sPayment, $aPendingIds)
  1963. {
  1964. $aPendingIds = array_values(array_unique(array_map('intval', $aPendingIds)));
  1965. if(empty($aPendingIds))
  1966. return [];
  1967. return $this->getPairs("SELECT `pending_id`, MIN(`ts`) AS `first` FROM `" . $this->_oConfig->CNF['TABLE_REVENUE'] . "` WHERE `payment` = :payment AND `pending_id` IN (" . $this->implode_escape($aPendingIds) . ") AND `kind` IN ('sale', 'renewal') GROUP BY `pending_id`", 'pending_id', 'first', ['payment' => $sPayment]);
  1968. }
  1969. /**
  1970. * Transactions of a list the ledger has as a charge (a sale, a renewal or a trial's
  1971. * registration): [transaction id, ...].
  1972. */
  1973. public function getRevenueCharged($sPayment, $aTransactionIds)
  1974. {
  1975. $aTransactionIds = array_values(array_unique(array_map('intval', $aTransactionIds)));
  1976. if(empty($aTransactionIds))
  1977. return [];
  1978. return array_map('intval', $this->getColumn("SELECT DISTINCT `transaction_id` FROM `" . $this->_oConfig->CNF['TABLE_REVENUE'] . "` WHERE `payment` = :payment AND `transaction_id` IN (" . $this->implode_escape($aTransactionIds) . ") AND `kind` IN ('sale', 'renewal', 'trial')", ['payment' => $sPayment]));
  1979. }
  1980. /**
  1981. * A charge got its kind once its subscription's trial was known (`kind` is part of the
  1982. * unique key: a transaction has one charge row, so the change never meets another row).
  1983. */
  1984. public function updateRevenueKind($iId, $sKind)
  1985. {
  1986. return (int)$this->query("UPDATE IGNORE `" . $this->_oConfig->CNF['TABLE_REVENUE'] . "` SET `kind` = :kind WHERE `id` = :id", ['id' => (int)$iId, 'kind' => $sKind]);
  1987. }
  1988. /**
  1989. * Paid charges of an order in the ledger from a time on (a trial's registration is not
  1990. * one): first and last time, and how many (the rows of one charge share their time).
  1991. */
  1992. public function getRevenuePaid($sPayment, $iPendingId, $iSince = 0)
  1993. {
  1994. return $this->getRow("SELECT IFNULL(MIN(`ts`), 0) AS `first`, IFNULL(MAX(`ts`), 0) AS `last`, COUNT(DISTINCT `ts`) AS `count` FROM `" . $this->_oConfig->CNF['TABLE_REVENUE'] . "` WHERE `payment` = :payment AND `pending_id` = :pending AND `kind` IN ('sale', 'renewal') AND `ts` >= :since", [
  1995. 'payment' => $sPayment,
  1996. 'pending' => (int)$iPendingId,
  1997. 'since' => (int)$iSince,
  1998. ]);
  1999. }
  2000. /**
  2001. * Give an order's rows that have no visit yet the buyer's platform, visit and sources.
  2002. */
  2003. public function updateRevenueContext($sPayment, $iPendingId, $aSet)
  2004. {
  2005. $aBindings = ['payment' => $sPayment, 'pending' => (int)$iPendingId];
  2006. $aSql = [];
  2007. foreach($aSet as $sField => $mixedValue) {
  2008. $aSql[] = '`' . $sField . '` = :f_' . $sField;
  2009. $aBindings['f_' . $sField] = $mixedValue;
  2010. }
  2011. return (int)$this->query("UPDATE `" . $this->_oConfig->CNF['TABLE_REVENUE'] . "` SET " . implode(', ', $aSql) . " WHERE `payment` = :payment AND `pending_id` = :pending AND `session_id` = 0", $aBindings);
  2012. }
  2013. /**
  2014. * The traffic source of a member's first visit, and of the latest one that started by a time.
  2015. *
  2016. * @return array ['first' => row or [], 'last' => row or []] with channel, referrer_host, utm_campaign
  2017. */
  2018. public function getSessionSources($iProfileId, $iTime)
  2019. {
  2020. $sSql = "SELECT `channel`, `referrer_host`, `utm_campaign` FROM `" . $this->_oConfig->CNF['TABLE_SESSIONS'] . "` WHERE `profile_id` = :profile AND `started` <= :time ORDER BY `started` %s LIMIT 1";
  2021. $aBindings = ['profile' => (int)$iProfileId, 'time' => (int)$iTime];
  2022. return [
  2023. 'first' => $this->getRow(sprintf($sSql, 'ASC'), $aBindings),
  2024. 'last' => $this->getRow(sprintf($sSql, 'DESC'), $aBindings),
  2025. ];
  2026. }
  2027. /**
  2028. * A deleted profile: its money stays in the ledger, its visit and traffic sources don't.
  2029. */
  2030. public function forgetRevenueProfile($iProfileId)
  2031. {
  2032. return (int)$this->query("UPDATE `" . $this->_oConfig->CNF['TABLE_REVENUE'] . "` SET `session_id` = 0, `first_channel` = '', `first_referrer_host` = '', `first_utm_campaign` = '', `last_channel` = '', `last_referrer_host` = '', `last_utm_campaign` = '' WHERE `client_id` = :profile", ['profile' => (int)$iProfileId]);
  2033. }
  2034. /**
  2035. * Rows the import of past payments wrote (they have no platform).
  2036. */
  2037. public function getRevenueImportedCount($sPayment)
  2038. {
  2039. return (int)$this->getOne("SELECT COUNT(*) FROM `" . $this->_oConfig->CNF['TABLE_REVENUE'] . "` WHERE `payment` = :payment AND `platform` = ''", ['payment' => $sPayment]);
  2040. }
  2041. public function hasRevenue($sPayment)
  2042. {
  2043. return (bool)$this->getOne("SELECT 1 FROM `" . $this->_oConfig->CNF['TABLE_REVENUE'] . "` WHERE `payment` = :payment LIMIT 1", ['payment' => $sPayment]);
  2044. }
  2045. public function getSubscriptionRow($sPayment, $iPendingId)
  2046. {
  2047. return $this->getRow("SELECT * FROM `" . $this->_oConfig->CNF['TABLE_SUBSCRIPTIONS'] . "` WHERE `payment` = :payment AND `pending_id` = :pending", [
  2048. 'payment' => $sPayment,
  2049. 'pending' => (int)$iPendingId,
  2050. ]);
  2051. }
  2052. /**
  2053. * Start and trial (days) of the subscriptions with a trial among some orders:
  2054. * [pending id => ['started' => time, 'trial' => days]].
  2055. */
  2056. public function getSubscriptionTrials($sPayment, $aPendingIds)
  2057. {
  2058. $aPendingIds = array_values(array_unique(array_map('intval', $aPendingIds)));
  2059. if(empty($aPendingIds))
  2060. return [];
  2061. return $this->getAllWithKey("SELECT `pending_id`, `started`, `trial` FROM `" . $this->_oConfig->CNF['TABLE_SUBSCRIPTIONS'] . "` WHERE `payment` = :payment AND `pending_id` IN (" . $this->implode_escape($aPendingIds) . ") AND `trial` > 0", 'pending_id', ['payment' => $sPayment]);
  2062. }
  2063. /**
  2064. * A new subscription, or new terms for one the ledger has (it keeps its start, payments
  2065. * and cancellation).
  2066. */
  2067. public function saveSubscriptionRow($aRow)
  2068. {
  2069. $aBindings = [];
  2070. foreach($aRow as $sField => $mixedValue)
  2071. $aBindings[$sField] = $mixedValue;
  2072. $aUpdate = [];
  2073. foreach(['subscription_id', 'client_id', 'seller_id', 'module_id', 'module', 'item_id', 'provider', 'period', 'period_unit', 'trial', 'amount_default', 'mrr_default'] as $sField)
  2074. if(array_key_exists($sField, $aBindings))
  2075. $aUpdate[] = '`' . $sField . '` = VALUES(`' . $sField . '`)';
  2076. if(array_key_exists('status', $aBindings))
  2077. $aUpdate[] = "`status` = IF(`canceled` > 0, `status`, VALUES(`status`))";
  2078. $sFields = '`' . implode('`, `', array_keys($aBindings)) . '`';
  2079. $sValues = ':' . implode(', :', array_keys($aBindings));
  2080. return $this->query("INSERT INTO `" . $this->_oConfig->CNF['TABLE_SUBSCRIPTIONS'] . "` (" . $sFields . ") VALUES (" . $sValues . ") ON DUPLICATE KEY UPDATE " . implode(', ', $aUpdate), $aBindings) !== false;
  2081. }
  2082. /**
  2083. * A subscription read from the payment module's history: fills what the ledger doesn't
  2084. * know yet and leaves what it recorded live.
  2085. */
  2086. public function importSubscriptionRow($aRow)
  2087. {
  2088. $aBindings = [];
  2089. foreach($aRow as $sField => $mixedValue)
  2090. $aBindings[$sField] = $mixedValue;
  2091. $sFields = '`' . implode('`, `', array_keys($aBindings)) . '`';
  2092. $sValues = ':' . implode(', :', array_keys($aBindings));
  2093. return $this->query("INSERT INTO `" . $this->_oConfig->CNF['TABLE_SUBSCRIPTIONS'] . "` (" . $sFields . ") VALUES (" . $sValues . ") ON DUPLICATE KEY UPDATE
  2094. `first_paid` = IF(`first_paid` = 0, VALUES(`first_paid`), LEAST(`first_paid`, IF(VALUES(`first_paid`) > 0, VALUES(`first_paid`), `first_paid`))),
  2095. `last_paid` = GREATEST(`last_paid`, VALUES(`last_paid`)),
  2096. `paid_count` = GREATEST(`paid_count`, VALUES(`paid_count`)),
  2097. `reason` = IF(`canceled` = 0, VALUES(`reason`), `reason`),
  2098. `status` = IF(`canceled` = 0 AND VALUES(`canceled`) > 0, VALUES(`status`), `status`),
  2099. `canceled` = IF(`canceled` = 0, VALUES(`canceled`), `canceled`)", $aBindings) !== false;
  2100. }
  2101. public function updateSubscriptionRow($sPayment, $iPendingId, $aSet)
  2102. {
  2103. $aBindings = ['payment' => $sPayment, 'pending' => (int)$iPendingId];
  2104. $aSql = [];
  2105. foreach($aSet as $sField => $mixedValue) {
  2106. $aSql[] = '`' . $sField . '` = :f_' . $sField;
  2107. $aBindings['f_' . $sField] = $mixedValue;
  2108. }
  2109. return (int)$this->query("UPDATE `" . $this->_oConfig->CNF['TABLE_SUBSCRIPTIONS'] . "` SET " . implode(', ', $aSql) . " WHERE `payment` = :payment AND `pending_id` = :pending", $aBindings);
  2110. }
  2111. /**
  2112. * Charges of a subscription were recorded (`$iCount` charges, from `$iFirst` to `$iLast`):
  2113. * it is active (unless cancelled) since its first.
  2114. */
  2115. public function addSubscriptionPayments($sPayment, $iPendingId, $iCount, $iFirst, $iLast)
  2116. {
  2117. return (int)$this->query("UPDATE `" . $this->_oConfig->CNF['TABLE_SUBSCRIPTIONS'] . "` SET `first_paid` = IF(`first_paid` = 0, :first, LEAST(`first_paid`, :first)), `last_paid` = GREATEST(`last_paid`, :last), `paid_count` = `paid_count` + :count, `status` = IF(`canceled` > 0, `status`, 'active') WHERE `payment` = :payment AND `pending_id` = :pending", [
  2118. 'payment' => $sPayment,
  2119. 'pending' => (int)$iPendingId,
  2120. 'count' => (int)$iCount,
  2121. 'first' => (int)$iFirst,
  2122. 'last' => (int)$iLast,
  2123. ]);
  2124. }
  2125. /**
  2126. * Gross (sales and renewals), refunds (refunds and reversals, positive), net, orders,
  2127. * paying customers, renewals and movements without an exchange rate in a period.
  2128. *
  2129. * @param array $aFilter report filter with `payment`, the payment module to read
  2130. * @param int $iCash 1 for money, 0 for orders paid with credits, null for both
  2131. */
  2132. public function getRevenueTotals($aFilter, $iCash = 1)
  2133. {
  2134. $aBindings = [];
  2135. $sWhere = $this->_getRevenueWhere($aFilter, $iCash, $aBindings);
  2136. return $this->getRow("SELECT " . $this->_getRevenueSums() . ", COUNT(*) AS `movements` FROM `" . $this->_oConfig->CNF['TABLE_REVENUE'] . "` AS `r` WHERE " . $sWhere, $aBindings);
  2137. }
  2138. /**
  2139. * The same sums per time bucket: rows keyed by bucket start.
  2140. */
  2141. public function getRevenueSeries($aFilter, $iCash = 1)
  2142. {
  2143. $aBindings = [];
  2144. $sWhere = $this->_getRevenueWhere($aFilter, $iCash, $aBindings);
  2145. $sBucket = $this->_getBucketSql('`r`.`ts`', $aFilter['bucket'], $aFilter['offset'], $aFilter['starts'], $aBindings);
  2146. return $this->getAllWithKey("SELECT " . $sBucket . " AS `bucket`, " . $this->_getRevenueSums() . " FROM `" . $this->_oConfig->CNF['TABLE_REVENUE'] . "` AS `r` WHERE " . $sWhere . " GROUP BY `bucket` ORDER BY `bucket`", 'bucket', $aBindings);
  2147. }
  2148. /**
  2149. * Net and gross revenue, customers and orders per value of a ledger column, the largest first.
  2150. */
  2151. public function getRevenueSplit($aFilter, $sDimension, $iLimit, $iCash = 1)
  2152. {
  2153. if(!isset($this->_aRevenueDimensions[$sDimension]))
  2154. return [];
  2155. $aBindings = [];
  2156. $sWhere = $this->_getRevenueWhere($aFilter, $iCash, $aBindings);
  2157. $sColumn = '`r`.`' . $this->_aRevenueDimensions[$sDimension] . '`';
  2158. return $this->getAll("SELECT " . $sColumn . " AS `value`, " . $this->_getRevenueSums() . " FROM `" . $this->_oConfig->CNF['TABLE_REVENUE'] . "` AS `r` WHERE " . $sWhere . " GROUP BY " . $sColumn . " ORDER BY `net` DESC LIMIT " . max(1, min(1000, (int)$iLimit)), $aBindings);
  2159. }
  2160. /**
  2161. * Items that brought the most net revenue, with units sold and orders.
  2162. */
  2163. public function getRevenueTopItems($aFilter, $iLimit)
  2164. {
  2165. $aBindings = [];
  2166. $sWhere = $this->_getRevenueWhere($aFilter, 1, $aBindings);
  2167. return $this->getAll("SELECT `r`.`module_id`, MAX(`r`.`module`) AS `module`, `r`.`item_id`, IFNULL(SUM(`r`.`amount_default`), 0) AS `net`, IFNULL(SUM(IF(`r`.`kind` = 'sale', `r`.`item_count`, 0)), 0) AS `units`, COUNT(DISTINCT IF(`r`.`kind` = 'sale', `r`.`pending_id`, NULL)) AS `orders`, SUM(`r`.`kind` = 'renewal') AS `renewals` FROM `" . $this->_oConfig->CNF['TABLE_REVENUE'] . "` AS `r` WHERE " . $sWhere . " GROUP BY `r`.`module_id`, `r`.`item_id` ORDER BY `net` DESC LIMIT " . max(1, min(100, (int)$iLimit)), $aBindings);
  2168. }
  2169. /**
  2170. * Revenue the site sold itself (`$iSiteAdmin`, 0 when unknown), revenue of other sellers,
  2171. * and commission invoices paid to the site (rows of the payment module's own items).
  2172. */
  2173. public function getRevenueSellers($aFilter, $iSiteAdmin)
  2174. {
  2175. $aBindings = ['admin' => (int)$iSiteAdmin, 'payment_module' => isset($aFilter['payment']) ? $aFilter['payment'] : ''];
  2176. $sWhere = $this->_getRevenueWhere($aFilter, 1, $aBindings);
  2177. return $this->getRow("SELECT IFNULL(SUM(IF(`r`.`module` <> :payment_module AND `r`.`seller_id` = :admin, `r`.`amount_default`, 0)), 0) AS `site`, IFNULL(SUM(IF(`r`.`module` <> :payment_module AND `r`.`seller_id` <> :admin, `r`.`amount_default`, 0)), 0) AS `vendors`, IFNULL(SUM(IF(`r`.`module` = :payment_module, `r`.`amount_default`, 0)), 0) AS `commissions`, COUNT(DISTINCT IF(`r`.`module` <> :payment_module AND `r`.`seller_id` <> :admin, `r`.`seller_id`, NULL)) AS `vendor_count` FROM `" . $this->_oConfig->CNF['TABLE_REVENUE'] . "` AS `r` WHERE " . $sWhere, $aBindings);
  2178. }
  2179. /**
  2180. * Net revenue of members over all the ledger, trials' registrations aside (they took no
  2181. * money): [profile id => amount].
  2182. */
  2183. public function getLifetimeValues($aProfileIds, $sPayment)
  2184. {
  2185. $aProfileIds = array_values(array_unique(array_filter(array_map('intval', $aProfileIds))));
  2186. if(empty($aProfileIds))
  2187. return [];
  2188. return $this->getPairs("SELECT `client_id`, IFNULL(SUM(`amount_default`), 0) AS `ltv` FROM `" . $this->_oConfig->CNF['TABLE_REVENUE'] . "` WHERE `cash` = 1 AND `payment` = :payment AND `kind` <> 'trial' AND `client_id` IN (" . $this->implode_escape($aProfileIds) . ") GROUP BY `client_id`", 'client_id', 'ltv', ['payment' => $sPayment]);
  2189. }
  2190. /**
  2191. * Subscriptions that were paying or in trial at some time in a period: paid or started
  2192. * with a trial before its end, and not cancelled or lapsed before its start.
  2193. *
  2194. * @param array $aEnd when an unpaid subscription stops paying: unit_seconds [period unit =>
  2195. * seconds], unpaid (the status), grace (seconds); none to read lapsed ones too
  2196. */
  2197. public function getSubscriptionRows($aFilter, $aEnd = [])
  2198. {
  2199. $aBindings = ['from' => (int)$aFilter['from'], 'to' => (int)$aFilter['to'], 'payment' => isset($aFilter['payment']) ? $aFilter['payment'] : ''];
  2200. $sPlatform = '';
  2201. if($aFilter['platform'] !== '') {
  2202. $sPlatform = ' AND `platform` = :platform';
  2203. $aBindings['platform'] = $aFilter['platform'];
  2204. }
  2205. // an unpaid subscription stopped paying when its last paid period and the grace time
  2206. // ended, without a cancellation: those that stopped before the period stay out, or
  2207. // every lapsed one would be read forever (see BxAnalyticsReports::_getSubscriptionEnd)
  2208. $sLapsed = '';
  2209. if(!empty($aEnd['unit_seconds'])) {
  2210. $aUnits = [];
  2211. foreach($aEnd['unit_seconds'] as $sUnit => $iSeconds) {
  2212. $sBinding = 'unit' . count($aUnits);
  2213. $aUnits[] = 'WHEN :' . $sBinding . ' THEN ' . (int)$iSeconds;
  2214. $aBindings[$sBinding] = $sUnit;
  2215. }
  2216. $sLapsed = " AND NOT (`canceled` = 0 AND `status` = :unpaid AND `last_paid` > 0 AND `last_paid` + :grace + GREATEST(1, `period`) * (CASE `period_unit` " . implode(' ', $aUnits) . " ELSE 0 END) < :from)";
  2217. $aBindings['unpaid'] = $aEnd['unpaid'];
  2218. $aBindings['grace'] = (int)$aEnd['grace'];
  2219. }
  2220. return $this->getAll("SELECT `pending_id`, `module`, `item_id`, `period`, `period_unit`, `trial`, `mrr_default`, `started`, `first_paid`, `last_paid`, `canceled`, `status` FROM `" . $this->_oConfig->CNF['TABLE_SUBSCRIPTIONS'] . "` WHERE `payment` = :payment" . $sPlatform . " AND (`canceled` = 0 OR `canceled` >= :from)" . $sLapsed . " AND ((`first_paid` > 0 AND `first_paid` < :to) OR ((`trial` > 0 OR `status` = 'trial') AND `started` < :to))", $aBindings);
  2221. }
  2222. /**
  2223. * Why subscriptions were cancelled in a period: [reason => count], the most frequent first.
  2224. * `cancel` is the payment module's default for no reason, counted with the empty one.
  2225. */
  2226. public function getSubscriptionReasons($aFilter, $iLimit)
  2227. {
  2228. $aBindings = ['from' => (int)$aFilter['from'], 'to' => (int)$aFilter['to'], 'payment' => isset($aFilter['payment']) ? $aFilter['payment'] : ''];
  2229. $sPlatform = '';
  2230. if($aFilter['platform'] !== '') {
  2231. $sPlatform = ' AND `platform` = :platform';
  2232. $aBindings['platform'] = $aFilter['platform'];
  2233. }
  2234. return $this->getPairs("SELECT IF(`reason` = 'cancel', '', `reason`) AS `cause`, COUNT(*) AS `count` FROM `" . $this->_oConfig->CNF['TABLE_SUBSCRIPTIONS'] . "` WHERE `payment` = :payment" . $sPlatform . " AND `canceled` >= :from AND `canceled` < :to GROUP BY `cause` ORDER BY `count` DESC LIMIT " . max(1, min(100, (int)$iLimit)), 'cause', 'count', $aBindings);
  2235. }
  2236. protected function _getRevenueSums()
  2237. {
  2238. return "IFNULL(SUM(IF(`r`.`kind` IN ('sale', 'renewal'), `r`.`amount_default`, 0)), 0) AS `gross`, IFNULL(-SUM(IF(`r`.`kind` IN ('refund', 'reversal'), `r`.`amount_default`, 0)), 0) AS `refunds`, IFNULL(SUM(`r`.`amount_default`), 0) AS `net`, COUNT(DISTINCT IF(`r`.`kind` = 'sale', `r`.`pending_id`, NULL)) AS `orders`, COUNT(DISTINCT IF(`r`.`kind` IN ('sale', 'renewal'), `r`.`client_id`, NULL)) AS `customers`, COUNT(DISTINCT IF(`r`.`kind` = 'renewal', CONCAT(`r`.`pending_id`, ':', `r`.`ts`), NULL)) AS `renewals`, SUM(`r`.`kind` IN ('refund', 'reversal')) AS `refund_count`, IFNULL(SUM(`r`.`amount_default` IS NULL), 0) AS `unconverted`";
  2239. }
  2240. /**
  2241. * WHERE for ledger rows (alias `r`) of one payment module in the period, money or credits
  2242. * (null: both), on the filter's platform. A trial's registration took no money: no revenue
  2243. * sum, count or chart reads it.
  2244. */
  2245. protected function _getRevenueWhere($aFilter, $iCash, &$aBindings)
  2246. {
  2247. $aSql = ['`r`.`ts` >= :from', '`r`.`ts` < :to', '`r`.`payment` = :payment', "`r`.`kind` <> 'trial'"];
  2248. if($iCash !== null) {
  2249. array_unshift($aSql, '`r`.`cash` = :cash');
  2250. $aBindings['cash'] = (int)$iCash ? 1 : 0;
  2251. }
  2252. $aBindings['from'] = (int)$aFilter['from'];
  2253. $aBindings['to'] = (int)$aFilter['to'];
  2254. $aBindings['payment'] = isset($aFilter['payment']) ? (string)$aFilter['payment'] : '';
  2255. if($aFilter['platform'] !== '') {
  2256. $aSql[] = '`r`.`platform` = :platform';
  2257. $aBindings['platform'] = $aFilter['platform'];
  2258. }
  2259. return implode(' AND ', $aSql);
  2260. }
  2261. protected function _getKindsSql($aKinds, &$aBindings)
  2262. {
  2263. if(empty($aKinds))
  2264. return '';
  2265. $aIn = [];
  2266. foreach(array_values($aKinds) as $i => $sKind) {
  2267. $aIn[] = ':k' . $i;
  2268. $aBindings['k' . $i] = $sKind;
  2269. }
  2270. return ' AND `kind` IN (' . implode(', ', $aIn) . ')';
  2271. }
  2272. /*
  2273. * Content reports
  2274. */
  2275. public function getGrowth($sTableName, $sColumnAdded, $iDateFrom, $iDateTo)
  2276. {
  2277. $aBindings = array(
  2278. 'datefrom' => $iDateFrom,
  2279. 'dateto' => $iDateTo
  2280. );
  2281. $sQuery = "SELECT DATE(FROM_UNIXTIME(`" . $sColumnAdded . "`)) AS `period`, YEAR(FROM_UNIXTIME(`" . $sColumnAdded . "`)) AS `year`, COUNT(*) AS `count` FROM " . $sTableName . " WHERE `" . $sColumnAdded . "` >= :datefrom AND `" . $sColumnAdded . "` <= :dateto GROUP BY `period`, `year` ORDER BY `year`, `period` ASC";
  2282. return $this->getAll($sQuery, $aBindings);
  2283. }
  2284. public function getGrowthInitValue($sTableName, $sColumnAdded, $iDateFrom)
  2285. {
  2286. $aBindings = array(
  2287. 'datefrom' => $iDateFrom
  2288. );
  2289. $sQuery = "SELECT COUNT(*) AS `count` FROM " . $sTableName . " WHERE `" . $sColumnAdded . "` < :datefrom ";
  2290. return $this->getOne($sQuery, $aBindings);
  2291. }
  2292. public function getTopContentByLikes($sModuleName, $sTableName, $iDateFrom, $iDateTo)
  2293. {
  2294. $aBindings = array(
  2295. 'datefrom' => $iDateFrom,
  2296. 'dateto' => $iDateTo
  2297. );
  2298. $sQuery = "SELECT `object_id`,'" . $sModuleName . "' AS `module`, COUNT(*) AS `value` FROM `" . $sTableName . "` WHERE `date` >= :datefrom AND `date` <= :dateto GROUP BY `object_id` ORDER BY COUNT(*) DESC LIMIT 0, " . intval(getParam('bx_analytics_items_count'));
  2299. return $this->getAll($sQuery, $aBindings);
  2300. }
  2301. public function getMostFollowedProfiles($sModuleName, $iDateFrom, $iDateTo)
  2302. {
  2303. $aBindings = array(
  2304. 'datefrom' => $iDateFrom,
  2305. 'dateto' => $iDateTo
  2306. );
  2307. $sQuery = "SELECT `sys_profiles`.`content_id` AS `object_id`, `sys_profiles`.`type` , COUNT(`sys_profiles_conn_subscriptions`.`id`) AS `value` FROM `sys_profiles_conn_subscriptions`
  2308. INNER JOIN `sys_profiles` ON `sys_profiles`.`id` = `sys_profiles_conn_subscriptions`.`content` WHERE sys_profiles.type='" . $sModuleName . "' AND `sys_profiles_conn_subscriptions`.`added` >= :datefrom AND `sys_profiles_conn_subscriptions`.`added` <= :dateto GROUP BY `sys_profiles`.`content_id`, `sys_profiles`.`type` ORDER BY COUNT(`sys_profiles_conn_subscriptions`.`id`) DESC LIMIT 0," . intval(getParam('bx_analytics_items_count'));
  2309. return $this->getAll($sQuery, $aBindings);
  2310. }
  2311. public function getMostActiveProfiles($sModuleName, $sContentModuleName, $sTableName, $sColumnAuthor, $sColumnAdded, $iDateFrom, $iDateTo)
  2312. {
  2313. $aBindings = array(
  2314. 'datefrom' => $iDateFrom,
  2315. 'dateto' => $iDateTo
  2316. );
  2317. $sQuery = "SELECT `sys_profiles`.`content_id` AS `object_id`, `sys_profiles`.`type`, '" . $sContentModuleName . "' , COUNT(`" . $sTableName . "`.`id`) AS `value` FROM `" . $sTableName . "`
  2318. INNER JOIN `sys_profiles` ON `sys_profiles`.`id` = `" . $sTableName . "`.`" . $sColumnAuthor . "` WHERE sys_profiles.type='" . $sModuleName . "' AND `" . $sTableName . "`.`" . $sColumnAdded . "` >= :datefrom AND `" . $sTableName . "`.`" . $sColumnAdded . "` <= :dateto GROUP BY `sys_profiles`.`content_id`, `sys_profiles`.`type` ORDER BY COUNT(`" . $sTableName . "`.`id`) DESC LIMIT 0," . intval(getParam('bx_analytics_items_count'));
  2319. return $this->getAll($sQuery, $aBindings);
  2320. }
  2321. }
  2322. /** @} */