Rodolfo Berrios 2021-10-19 11:24:51 -03:00
parent f668d7a918
commit 5cbde43928
No known key found for this signature in database
GPG Key ID: D3AAC2481DBDD9FE
7 changed files with 0 additions and 1518 deletions

File diff suppressed because it is too large Load Diff

View File

@ -161,7 +161,6 @@ class User
if (!is_array($values)) { if (!is_array($values)) {
throw new DBException('Expecting array values, ' . gettype($values) . ' given in ' . __METHOD__, 100); throw new DBException('Expecting array values, ' . gettype($values) . ' given in ' . __METHOD__, 100);
} }
// TODO: Role handler (for importer)
if (!$values['date']) { if (!$values['date']) {
$values['date'] = G\datetime(); $values['date'] = G\datetime();
} }

View File

@ -46,7 +46,6 @@ $route = function ($handler) {
'albums' => _s('Albums'), 'albums' => _s('Albums'),
'users' => _s('Users'), 'users' => _s('Users'),
'settings' => _s('Settings'), 'settings' => _s('Settings'),
'bulk' => _s('Bulk importer'),
]; ];
$default_route = 'stats'; $default_route = 'stats';

View File

@ -1,40 +0,0 @@
<?php
/* --------------------------------------------------------------------
This file is part of Chevereto Free.
https://chevereto.com/free
(c) Rodolfo Berrios <rodolfo@chevereto.com>
For the full copyright and license information, please view the LICENSE
file that was distributed with this source code.
--------------------------------------------------------------------- */
$route = function ($handler) {
try {
if (!CHV\Login::isAdmin()) {
$this->template = 'request-denied';
return;
}
// Allow 3 levels only -> /importer-jobs/X/process
if ($handler->isRequestLevel(4)) {
return $handler->issue404();
}
if (is_null($handler->request[0]) || is_null($handler->request[1])) {
return $handler->issue404();
}
$filepath = G_ROOT_PATH . sprintf('app/importer/jobs/%1$s/%2$s.txt', $handler->request[0], $handler->request[1]);
if (!file_exists($filepath)) {
return $handler->issue404();
}
if (!headers_sent()) {
header('Content-Type: text/plain');
}
readfile($filepath);
exit;
} catch (Exception $e) {
G\exception_to_error($e);
}
};

View File

@ -31,12 +31,6 @@ $route = function ($handler) {
$doing = 'deny'; $doing = 'deny';
} }
if (in_array($doing, ['importAdd', 'importStats', 'importProcess', 'importEdit', 'importDelete']) && $logged_user['is_admin'] == false) {
throw new Exception(_s('Request denied'), 400);
} else {
$import = new CHV\Import();
}
switch ($doing) { switch ($doing) {
case 'deny': case 'deny':
throw new Exception(_s('Request denied'), 403); throw new Exception(_s('Request denied'), 403);
@ -1342,33 +1336,6 @@ $route = function ($handler) {
} }
break; break;
// Adds the importer job (path+options)
case 'importAdd':
if ($_REQUEST['path'] == false) {
throw new Exception('Missing path parameter', 100);
}
$import->path = $_REQUEST['path'];
if ($_REQUEST['options'] != false) {
$import->options = $_REQUEST['options'];
}
$import->add();
$import->get();
$json_array['status_code'] = 200;
$json_array['import'] = $import->parsedImport;
break;
// Common operations
case 'importStats':
case 'importProcess':
case 'importEdit':
case 'importDelete':
case 'importReset':
case 'importResume':
if ($_REQUEST['id'] == false) {
throw new Exception('Missing id parameter', 100);
}
$import->id = (int) $_REQUEST['id'];
$import->get();
break;
case 'toggleTone': case 'toggleTone':
if (!$logged_user) { if (!$logged_user) {
throw new Exception('Invalid request', 403); throw new Exception('Invalid request', 403);
@ -1404,51 +1371,6 @@ $route = function ($handler) {
throw new Exception(!G\check_value($doing) ? 'empty action' : 'invalid action', !G\check_value($doing) ? 0 : 1); throw new Exception(!G\check_value($doing) ? 'empty action' : 'invalid action', !G\check_value($doing) ? 0 : 1);
break; break;
} }
if (isset($import->id)) {
switch ($doing) {
// Check the importer stats (id)
case 'importStats':
$json_array['status_code'] = 200;
$json_array['import'] = $import->parsedImport;
break;
// Issue/Resume import operation (id+thread)
case 'importProcess':
session_write_close();
$import->thread = (int) $_REQUEST['thread'] ?: 1;
$import->process();
$json_array['status_code'] = 200;
break;
// Edit import job (id,values)
case 'importEdit':
if ($_REQUEST['values'] == false) {
throw new Exception('Missing values parameter', 101);
}
if (is_array($_REQUEST['values']) == false) {
throw new Exception('Expecting array values', 102);
}
$import->edit($_REQUEST['values']);
$import->get();
$json_array['import'] = $import->parsedImport;
$json_array['status_code'] = 200;
break;
case 'importReset':
$import->reset();
$json_array['import'] = $import->parsedImport;
$json_array['status_code'] = 200;
break;
case 'importResume':
$import->resume();
$json_array['import'] = $import->parsedImport;
$json_array['status_code'] = 200;
break;
case 'importDelete':
$import->delete();
$json_array['status_code'] = 200;
$json_array['import'] = $import->parsedImport;
break;
}
}
// Inject any missing status_code
if (isset($json_array['success']) and !isset($json_array['status_code'])) { if (isset($json_array['success']) and !isset($json_array['status_code'])) {
$json_array['status_code'] = 200; $json_array['status_code'] = 200;
} }

View File

@ -122,352 +122,6 @@ function free_version_warning($wrap=true)
<?php <?php
break; break;
case 'bulk':
?>
<div class="header header--centering default-margin-bottom">
<h1><?php _se('Bulk importer'); ?></h1>
</div>
<div class="text-content">
<p><?php _se('This tool allows to mass add content to your website by pointing a system path with the content you want to import. It supports the addition of users, albums, and images using a folder based structure.'); ?> <?php _se('Check the %s for more information about this feature.', get_docs_link('features/bulk-content-importer.html', _s('documentation'))); ?></p>
</div>
<div class ="text-content margin-bottom-20">
<h3>🤖 <?php _se('Automatic importing'); ?></h3>
<p><?php _se('The system automatically parses any content by a continuous observation of the %path% path.', ['%path%' => '<b>' . G_ROOT_PATH . 'importing</b>']); ?> <?php _se('Completed jobs will be automatically re-started after %n %m.', [
'%n' => '1',
'%m' => _n('minute', 'minutes', '1')
]); ?> <?php _se('Reset to clear stats and logs.'); ?></p>
</div>
<div data-content="dashboard-imports" class="margin-top-0 margin-bottom-20">
<?php
$statusesDisplay = [
'queued' => _s('Queued'),
'working' => _s('Working'),
'paused' => _s('Paused'),
'canceled' => _s('Canceled'),
'completed' => _s('Completed'),
];
if ($continuous = CHV\Import::getContinuous()) {
foreach ($continuous as $v) {
$boxTpl = '<div class="importing phone-c1 phablet-c1 c8 fluid-column display-inline-block" data-status="%status%" data-id="%id%" data-object="%object%" data-errors="%errors%" data-started="%started%">
<h3 class="margin-bottom-5">Path <b title="%path%">%pathRelative%</b></h3>
<span data-content="log-errors" class="icon icon-warning color-red position-absolute top-10 right-10"></span>
<div data-content="pop-selection" class="pop-btn">
<span class="pop-btn-text">' . _s('Actions') . '<span class="arrow-down"></span></span>
<div class="pop-box arrow-box arrow-box-top anchor-left">
<div class="pop-box-inner pop-box-menu">
<ul>
<li data-action="reset"><a>' . _s('Reset') . '</a></li>
<li data-action="pause"><a>' . _s('Pause') . '</a></li>
<li data-action="resume"><a>' . _s('Resume') . '</a></li>
<li data-content="log-process"><a href="' . G\get_base_url('importer-jobs/%id%/process') . '" target="_blank">' . _s('Process log') . '</a></li>
<li data-content="log-errors"><a href="' . G\get_base_url('importer-jobs/%id%/errors') . '" target="_blank"><span class="icon icon-warning color-red"></span> ' . _s('Errors') . '</a></li>
</ul>
</div>
</div>
</div>
<div class="importing-stats">
<span class="figure"><span data-content="images">%images%</span> images</span>
<span class="figure"><span data-content="albums">%albums%</span> albums</span>
<span class="figure"><span data-content="users">%users%</span> users</span>
</div>
<div class="importing-status">' . _s('Status') . ': <span data-content="status">%displayStatus%</span></div>
<div class="importing-status">@<span data-content="dateTime">%dateTime%</span> UTC</div>
<div class="loading-container position-absolute right-10 bottom-10"><div class="loading"></div></div>
</div>';
echo strtr($boxTpl, [
'%id%' => $v['id'],
'%dateTime%' => $v['time_updated'],
'%object%' => htmlspecialchars(json_encode($v), ENT_QUOTES, 'UTF-8'),
'%status%' => $v['status'],
'%parse%' => $v['options']['root'],
'%shortParse%' => $v['options']['root'][0],
'%displayStatus%' => $statusesDisplay[$v['status']],
'%path%' => $v['path'],
'%pathRelative%' => '.' . G\absolute_to_relative($v['path']),
'%users%' => $v['users'] ?: 0,
'%images%' => $v['images'] ?: 0,
'%albums%' => $v['albums'] ?: 0,
'%errors%' => $v['errors'] ?: 0,
'%started%' => $v['started'] ?: 0,
]);
}
} ?>
</div>
<div class="margin-bottom-40">
<div class="margin-bottom-10"><?php _se('The system works with a scheduled command to continuously process the importing. It requires a crontab entry.'); ?></div>
<div class="margin-bottom-10">
<code class="code">* * * * * IS_CRON=1 THREAD_ID=1 /usr/bin/php <?php echo G_ROOT_PATH . 'importing.php'; ?> >/dev/null 2>&1</code>
</div>
<div class="margin-bottom-10"><?php _se('You can run the command in parallel by changing the integer value of %s%.', ['%s%' => '<code>THREAD_ID</code>']); ?></div>
<div class="margin-bottom-10">
<span class="highlight padding-5 display-inline-block"> <?php _se('All file-system permissions must be granted for the crontab user at %path%', ['%path%' => G_ROOT_PATH . 'importing']); ?></span>
</div>
</div>
</div>
<div data-modal="modal-add-import" class="hidden" data-submit-fn="CHV.fn.import.add.submit" data-ajax-deferred="CHV.fn.import.add.deferred">
<span class="modal-box-title"><?php _se('Add import job'); ?></span>
<div class="modal-form">
<?php G\Render\include_theme_file('snippets/form_import_add'); ?>
</div>
</div>
<div data-modal="modal-process-import" data-prompt="skip" class="hidden" data-load-fn="CHV.fn.import.process.load" data-submit-fn="CHV.fn.import.process.submit" data-ajax-deferred="CHV.fn.import.process.deferred">
<span class="modal-box-title"><?php _se('Process import'); ?></span>
<div class="modal-form">
<?php G\Render\include_theme_file('snippets/form_import_process'); ?>
</div>
</div>
<?php
if ($imports = CHV\Import::getOneTime()) {
foreach ($imports as $k => &$v) {
if ($v['status'] != 'working') {
continue;
}
$then = strtotime($v['time_updated']);
$now = strtotime(G\datetimegmt());
if ($now > ($then + 300)) { // 5 min
$v['status'] = 'paused';
CHV\DB::update('imports', ['status' => 'paused'], ['id' => $v['id']]);
}
}
$imports = array_reverse($imports);
}
$rowTpl = '<li data-status="%status%" data-id="%id%" data-object="%object%" data-errors="%errors%" data-started="%started%">
<span class="fluid-column c2 col-2-max display-table-cell padding-right-10">%id%</span>
<span class="fluid-column c2 col-2-max display-table-cell padding-right-10 text-transform-uppercase" title="' . _s('Top level folders as %s', '%parse%') . '">%shortParse%</span>
<span class="fluid-column c3 col-3-max display-table-cell padding-right-10">
<span class="icon icon-warning2 color-red" data-result="error"></span>
<span class="icon icon-checkmark-circle color-green" data-result="success"></span>
<span class="status-text">%displayStatus%</span>
</span>
<span class="fluid-column c7 col-7-max col-7-min display-table-cell padding-right-10 text-overflow-ellipsis phone-display-block" title="%path%">%path%</span>
<span class="fluid-column c2 display-table-cell padding-right-10"><span>%users%</span><span class="table-li--mobile-display"> ' . _s('Users') . '</span></span>
<span class="fluid-column c2 display-table-cell padding-right-10"><span>%albums%</span><span class="table-li--mobile-display"> ' . _s('Albums') . '</span></span>
<span class="fluid-column c3 display-table-cell padding-right-10"><span>%images%</span><span class="table-li--mobile-display"> ' . _s('Images') . '</span></span>
<div class="fluid-column c3 display-table-cell margin-bottom-0 phone-display-block">
<div class="loading"></div>
<div data-content="pop-selection" class="pop-btn">
<span class="pop-btn-text">' . _s('Actions') . '<span class="arrow-down"></span></span>
<div class="pop-box arrow-box arrow-box-top anchor-left">
<div class="pop-box-inner pop-box-menu">
<ul>
<li data-args="%id%" data-modal="form" data-target="modal-process-import"><a>' . _s('Process') . '</a></li>
<li data-action="pause"><a>' . _s('Pause') . '</a></li>
<li data-action="cancel"><a>' . _s('Cancel') . '</a></li>
<li data-content="log-process"><a href="' . G\get_base_url('importer-jobs/%id%/process') . '" target="_blank">' . _s('Process log') . '</a></li>
<li data-content="log-errors"><a href="' . G\get_base_url('importer-jobs/%id%/errors') . '" target="_blank">' . _s('Errors') . '</a></li>
<li data-args="%id%" data-confirm="' . _s('Do you really want to remove the import ID %s?', '%id%') . '" data-submit-fn="CHV.fn.import.delete.submit" data-ajax-deferred="CHV.fn.import.delete.deferred"><a>' . _s('Delete') . '</a></li>
</ul>
</div>
</div>
</div>
</div>
</li>';
$manualImportingClass = is_array($imports) == false ? ' hidden' : '';
?>
<div class="text-content">
<h3> <?php _se('Manual importing'); ?></h3>
<p><?php _se('The system will parse the contents of any available filesystem path.'); ?> <?php _se('These processes must be manually created and handled with the web browser tab open.'); ?> <a data-modal="form" data-target="modal-add-import"><?php _se('Add import job'); ?></a></p>
</div>
<ul data-content="dashboard-imports" class="tabbed-content-list table-li-hover table-li margin-top-20 margin-bottom-20<?php echo $manualImportingClass; ?>">
<li class="table-li-header phone-hide">
<span class="fluid-column c2 display-table-cell">ID</span>
<span class="fluid-column c2 display-table-cell"><?php _se('Parser'); ?></span>
<span class="fluid-column c3 display-table-cell"><?php _se('Status'); ?></span>
<span class="fluid-column c7 col-7-max col-7-min display-table-cell"><?php _se('Path'); ?></span>
<span class="fluid-column c2 display-table-cell"><?php _se('Users'); ?></span>
<span class="fluid-column c2 display-table-cell"><?php _se('Albums'); ?></span>
<span class="fluid-column c3 display-table-cell"><?php _se('Images'); ?></span>
<span class="fluid-column c3 display-table-cell">&nbsp;</span>
</li>
<?php
if (is_array($imports)) {
foreach ($imports as $k => $v) {
echo strtr($rowTpl, [
'%id%' => $v['id'],
'%object%' => htmlspecialchars(json_encode($v), ENT_QUOTES, 'UTF-8'),
'%status%' => $v['status'],
'%parse%' => $v['options']['root'],
'%shortParse%' => $v['options']['root'][0],
'%displayStatus%' => $statusesDisplay[$v['status']],
'%path%' => $v['path'],
'%users%' => $v['users'] ?: 0,
'%images%' => $v['images'] ?: 0,
'%albums%' => $v['albums'] ?: 0,
'%errors%' => $v['errors'] ?: 0,
'%started%' => $v['started'] ?: 0,
]);
}
}
?>
</ul>
<script>
$(document).ready(function() {
CHV.obj.import = {
working: {
// importId: {
// threads: {threadId: xhr},
// interval: interval,
// stats: xhr
// }
},
aborted: [],
boxTpl: <?php echo json_encode($boxTpl); ?>,
rowTpl: <?php echo json_encode($rowTpl); ?>,
importTr: {
'id': null,
// 'object' : null,
'status': null,
'parse': null,
'shortParse': null,
'displayStatus': null,
'path': null,
'users': null,
'images': null,
'albums': null,
'errors': null,
'started': null,
},
sel: {
root: "[data-content=dashboard-imports]",
header: ".table-li-header"
},
statusesDisplay: <?php echo json_encode($statusesDisplay); ?>
};
var updateContinuous = function( object) {
var $sel = $("[data-id=" + object.id + "]", CHV.obj.import.sel.root);
$sel.attr({
"data-status": object.status,
"data-object": JSON.stringify(object),
"data-errors": object.errors,
"data-started": object.started,
});
$("[data-content=images]", $sel).text(object.images);
$("[data-content=dateTime]", $sel).text(object.time_updated);
$("[data-content=users]", $sel).text(object.users);
$("[data-content=albums]", $sel).text(object.albums);
$("[data-content=status]", $sel).text(CHV.obj.import.statusesDisplay[object.status]);
};
$('.importing', '[data-content=dashboard-imports]').each(function(i, v) {
var id = $(this).data("id");
var $this = $(this);
CHV.obj.import.working[id] = {
threads: {},
interval: setInterval(function() {
var $loading = $this.find(".loading");
if($loading.html() !== "") {
$loading.removeClass("hidden");
} else {
PF.fn.loading.inline($loading, {
size: "small"
});
}
CHV.obj.import.working[id].stats = $.ajax({
type: "POST",
data: {
action: "importStats",
id: id
}
});
CHV.obj.import.working[id].stats.complete(function (XHR) {
var response = XHR.responseJSON;
if (response) {
updateContinuous(response.import);
}
$loading.addClass("hidden");
});
}, 10000),
stats: {}
};
});
$(document).on("click", CHV.obj.import.sel.root + " [data-action]", function() {
var $el = $(this).closest("[data-object]");
var $loading = $el.find(".loading");
var $actions = $el.find("[data-content=pop-selection]");
var localData = $el.data("object");
var backupData = $.extend({}, localData);
var action = $(this).data("action");
var data = {};
if (localData.id) {
data.id = localData.id;
}
if($el.is("li")) {
CHV.fn.import.process.abortAll(data.id);
}
switch (action) {
case "resume":
data.action = "importResume";
localData.status = "working";
break;
case "reset":
data.action = "importReset";
localData.status = "working";
break;
case "pause":
data.action = "importEdit";
localData.status = "paused";
data.values = {
status: localData.status
};
break;
case "cancel":
localData.status = "canceled";
data.action = "importEdit";
data.values = {
status: localData.status
};
break;
case "process":
localData.status = "working";
data.action = "importEdit";
data.values = {
status: localData.status
};
break;
default:
alert('null');
return;
break;
}
if($loading.html() !== "") {
$loading.removeClass("hidden");
} else {
PF.fn.loading.inline($loading, {
size: "small"
});
}
$actions.addClass("pointer-events-none");
$.ajax({
type: "POST",
data: data
}).complete(function(XHR) {
var response = XHR.responseJSON;
if (XHR.status == 200) {
var dataset = response.import;
if($el.is("li")) {
var $html = CHV.fn.import.parseTemplate(dataset);
$el.replaceWith($html);
if (action == "process") {
CHV.fn.import.process.deferred.success(XHR);
}
} else {
updateContinuous(response.import);
}
} else {
PF.fn.growl.call(response.error.message);
}
$loading.addClass("hidden");
$actions.removeClass("pointer-events-none");
});
});
});
</script>
<?php
break;
case 'images': case 'images':
case 'albums': case 'albums':

View File

@ -1,50 +0,0 @@
<?php
/* --------------------------------------------------------------------
This file is part of Chevereto Free.
https://chevereto.com/free
(c) Rodolfo Berrios <rodolfo@chevereto.com>
For the full copyright and license information, please view the LICENSE
file that was distributed with this source code.
--------------------------------------------------------------------- */
define('access', 'cli');
$isCron = getenv('IS_CRON');
if (!$isCron) {
header('HTTP/1.0 403 Forbidden');
die("403 Forbidden\n");
}
$threadID = getenv('THREAD_ID');
if (!$threadID) {
die("Missing thread id (int)\n");
}
if (!include_once('app/loader.php')) {
die("Can't find app/loader.php\n");
}
$loop = 1;
do {
try {
CHV\Import::refresh();
$jobs = CHV\Import::autoJobs();
if (!$jobs) {
echo "No jobs left.\n";
die(0);
}
$id = $jobs[0]['import_id'];
$import = new CHV\Import();
$import->id = $id;
$import->thread = (int) $threadID;
$import->process();
echo "Processed job id #$id\n";
$loop++;
} catch (Exception $e) {
echo $e->getMessage() . "\n";
die(255);
}
} while (CHV\isSafeToExecute());
echo "--\nLooped $loop times ~ /dashboard/bulk for stats \n";
die(0);