github
/
haproxy-wi
mirror of https://github.com/Aidaho12/haproxy-wi
1
0
Fork 0
Code Issues Releases Wiki Activity

v5.0.1.0 

Changelog: https://haproxy-wi.org/changelog.py#5_0_1
pull/304/head
Pavel Loginov 2021-03-04 11:53:27 +06:00
parent c433e2ed0a
commit 79f0a0a8bc
8 changed files with 399 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
app/funct.py
View File

@ -488,13 +488,12 @@ def get_stick_table(table):
def show_installation_output(error, output, service):
if error:
if "WARNING" not in error:
logging('localhost', error, haproxywi=1)
print('error: '+error)
if error and "WARNING" not in error:
logging('localhost', error, haproxywi=1)
print('error: '+error)
else:
for l in output:
if "FAILED" in l:
if "Traceback" in l or "FAILED" in l:
try:
l = l.split(':')[1]
l = l.split('"')[1]
@ -517,6 +516,7 @@ def install_haproxy(serv, **kwargs):
stats_password = sql.get_setting('stats_password')
proxy = sql.get_setting('proxy')
hapver = kwargs.get('hapver')
server_for_installing = kwargs.get('server')
ssh_port = 22
ssh_enable, ssh_user_name, ssh_user_password, ssh_key_name = return_ssh_keys_path(serv)
@ -546,8 +546,11 @@ def install_haproxy(serv, **kwargs):
" HOST=" + serv + " USER=" + ssh_user_name + " PASS=" + ssh_user_password + " KEY=" + ssh_key_name]
output, error = subprocess_execute(commands[0])
show_installation_output(error, output, 'HAProxy')
if server_for_installing:
service = server_for_installing + ' HAProxy'
else:
service = ' HAProxy'
show_installation_output(error, output, service)
os.system("rm -f %s" % script)
sql.update_haproxy(serv)
@ -583,7 +586,7 @@ def waf_install(serv):
sql.insert_waf_rules(serv)
def install_nginx(serv):
def install_nginx(serv, **kwargs):
import sql
script = "install_nginx.sh"
stats_user = sql.get_setting('nginx_stats_user')
@ -591,6 +594,7 @@ def install_nginx(serv):
stats_port = sql.get_setting('nginx_stats_port')
stats_page = sql.get_setting('nginx_stats_page')
config_path = sql.get_setting('nginx_config_path')
server_for_installing = kwargs.get('server')
proxy = sql.get_setting('proxy')
ssh_enable, ssh_user_name, ssh_user_password, ssh_key_name = return_ssh_keys_path(serv)
@ -616,8 +620,11 @@ def install_nginx(serv):
" HOST=" + serv + " USER=" + ssh_user_name + " PASS=" + ssh_user_password + " KEY=" + ssh_key_name]
output, error = subprocess_execute(commands[0])
show_installation_output(error, output, 'Nginx')
if server_for_installing:
service = server_for_installing + ' Nginx'
else:
service = ' Nginx'
show_installation_output(error, output, service)
os.system("rm -f %s" % script)
sql.update_nginx(serv)

57
app/options.py
View File

@ -567,7 +567,7 @@ if act == "overviewServers":
async def async_get_overviewServers(serv1, serv2, service):
if service == 'haproxy':
cmd = 'echo "show info" |nc %s %s -w 1|grep -e "node\|Nbproc\|Maxco\|MB\|Peers\|Nbthread"' % (serv2, sql.get_setting('haproxy_sock_port'))
cmd = 'echo "show info" |nc %s %s -w 1|grep -e "node\|Nbproc\|Maxco\|MB\|Nbthread"' % (serv2, sql.get_setting('haproxy_sock_port'))
out = funct.subprocess_execute(cmd)
out1 = ""
@ -952,22 +952,25 @@ if form.getvalue('master'):
os.system("cp scripts/%s ." % script)
if form.getvalue('hap') == "1":
funct.install_haproxy(master)
funct.install_haproxy(slave)
if form.getvalue('nginx') == "1":
funct.install_nginx(master)
funct.install_nginx(slave)
commands = ["chmod +x " + script + " && ./" + script + " PROXY=" + proxy_serv + " SSH_PORT=" + ssh_port +
" ETH=" + ETH + " IP=" + str(IP) + " MASTER=MASTER" + " SYN_FLOOD=" + syn_flood + " HOST=" + str(master) +
" USER=" + str(ssh_user_name) + " PASS=" + str(ssh_user_password) + " KEY=" + str(ssh_key_name)]
output, error = funct.subprocess_execute(commands[0])
funct.show_installation_output(error, output, 'Master Keepalived')
funct.show_installation_output(error, output, 'master Keepalived')
sql.update_keepalived(master)
if form.getvalue('master_slave'):
master = form.getvalue('master')
slave = form.getvalue('slave')
ETH = form.getvalue('interface')
IP = form.getvalue('vrrpip')
syn_flood = form.getvalue('syn_flood')
script = "install_keepalived.sh"
fullpath = funct.get_config_var('main', 'fullpath')
proxy = sql.get_setting('proxy')
ssh_enable, ssh_user_name, ssh_user_password, ssh_key_name = funct.return_ssh_keys_path(slave)
if ssh_enable == 0:
@ -977,17 +980,23 @@ if form.getvalue('master'):
for server in servers:
ssh_port = str(server[10])
if proxy is not None and proxy != '' and proxy != 'None':
proxy_serv = proxy
else:
proxy_serv = ''
os.system("cp scripts/%s ." % script)
commands = ["chmod +x " + script + " && ./" + script + " PROXY=" + proxy_serv + " SSH_PORT=" + ssh_port +
" ETH=" + ETH + " IP=" + IP + " MASTER=BACKUP" + " HOST=" + str(slave) +
" USER=" + str(ssh_user_name) + " PASS=" + str(ssh_user_password) + " KEY=" + str(ssh_key_name)]
output, error = funct.subprocess_execute(commands[0])
funct.show_installation_output(error, output, 'Slave Keepalived')
funct.show_installation_output(error, output, 'slave Keepalived')
os.system("rm -f %s" % script)
sql.update_server_master(master, slave)
sql.update_keepalived(master)
sql.update_keepalived(slave)
if form.getvalue('masteradd'):
@ -1021,7 +1030,7 @@ if form.getvalue('masteradd'):
output, error = funct.subprocess_execute(commands[0])
funct.show_installation_output(error, output, 'Master VRRP address')
funct.show_installation_output(error, output, 'master VRRP address')
ssh_enable, ssh_user_name, ssh_user_password, ssh_key_name = funct.return_ssh_keys_path(slave)
@ -1039,10 +1048,30 @@ if form.getvalue('masteradd'):
output, error = funct.subprocess_execute(commands[0])
funct.show_installation_output(error, output, 'Slave VRRP address')
funct.show_installation_output(error, output, 'slave VRRP address')
os.system("rm -f %s" % script)
if form.getvalue('master_slave_hap'):
master = form.getvalue('master_slave_hap')
slave = form.getvalue('slave')
server = form.getvalue('server')
if server == 'master':
funct.install_haproxy(master, server=server)
elif server == 'slave':
funct.install_haproxy(slave, server=server)
if form.getvalue('master_slave_nginx'):
master = form.getvalue('master_slave_nginx')
slave = form.getvalue('slave')
server = form.getvalue('server')
if server == 'master':
funct.install_nginx(master, server=server)
elif server == 'slave':
funct.install_nginx(slave, server=server)
if form.getvalue('install_grafana'):
script = "install_grafana.sh"
proxy = sql.get_setting('proxy')

6
app/scripts/ansible/roles/haproxy/tasks/main.yml
View File

@ -47,6 +47,7 @@
- http://repo1.haproxy-wi.org/haproxy-{{HAPVER}}.el{{ansible_facts['distribution_major_version']}}.x86_64.rpm
- socat
- rsyslog
- bind-utils
state: present
register: install_result1
when: (ansible_facts['os_family'] == "RedHat" or ansible_facts['os_family'] == 'CentOS') and ("'timed out' in install_result.stderr")
@ -176,7 +177,10 @@
ignore_errors: yes
no_log: True
debugger: never
when: (ansible_facts['os_family'] == "RedHat" or ansible_facts['os_family'] == 'CentOS') and ansible_facts.services["firewalld.service"]['state'] == "running"
when:
- '"firewalld" in ansible_facts.packages'
- ansible_facts['os_family'] == "RedHat" or ansible_facts['os_family'] == 'CentOS'
- ansible_facts.services["firewalld.service"]['state'] == "running"
with_items: [ "{{ STAT_PORT }}", "{{ SOCK_PORT }}" ]

6
app/scripts/ansible/roles/nginx.yml
View File

@ -11,7 +11,6 @@
package_facts:
manager: "auto"
- name: populate service facts
service_facts:
@ -72,7 +71,10 @@
ignore_errors: yes
no_log: True
debugger: never
when: (ansible_facts['os_family'] == "RedHat" or ansible_facts['os_family'] == 'CentOS') and ansible_facts.services["firewalld.service"]['state'] == "running"
when:
- '"firewalld" in ansible_facts.packages'
- ansible_facts['os_family'] == "RedHat" or ansible_facts['os_family'] == 'CentOS'
- ansible_facts.services["firewalld.service"]['state'] == "running"
- name: Open stat port for iptables

33
app/templates/ha.html
View File

@ -2,6 +2,11 @@
{% block content %}
{% from 'include/input_macros.html' import input, checkbox %}
<script src="/inc/users.js"></script>
<script src="/inc/ha.js"></script>
<link href="/inc/provisioning.css" rel="stylesheet">
<style>
p {margin: 0;}
</style>
<table class="overview">
<caption><h3>Create new HA cluster</h3></caption>
<tr class="overviewHead">
@ -85,11 +90,25 @@
<div class="add-note addName alert-info" style="width: inherit; margin-right: 15px;">
Read <a href="https://haproxy-wi.org/howto.py?howto=ha_cluster" title="How to create high available cluster" target="_blank">How to create high available cluster</a>
</div>
<script>
$( function() {
$( "select" ).selectmenu({
width: 180
});
});
</script>
<div id="server_creating" style="display: none;">
<ul style="padding: 20px 20px 0px 20px;font-size: 15px;">
<li id="creating-master" class="server-creating proccessing">Installing Master Keepalived...</li>
<li id="creating-slave" class="server-creating proccessing">Installing Slave Keepalived...</li>
<div id="haproxy_installing_div" style="display: none">
<li id="creating-haproxy-master" class="server-creating">Installing Master Haproxy...</li>
<li id="creating-haproxy-slave" class="server-creating">Installing Slave Haproxy...</li>
</div>
<div id="nginx_installing_div" style="display: none">
<li id="creating-nginx-master" class="server-creating">Installing Master Nginx...</li>
<li id="creating-nginx-slave" class="server-creating">Installing Slave Nginx...</li>
</div>
</ul>
<div id="wait-mess"></div>
<div id="created-mess" class="alert alert-success" style="display:none;"></div>
<div id="creating-error" class="alert alert-danger" style="display:none;"></div>
<div id="creating-warning" class="alert alert-warning" style="display:none;"></div>
<div class="progress-bar-striped">
<div id="creating-progress" style="width: 0%;"></div>
</div>
</div>
{% endblock %}

299
inc/ha.js Normal file
View File

@ -0,0 +1,299 @@
$( function() {
$( "select" ).selectmenu({
width: 180
});
var ipformat = /^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/;
$('#add-vrrp').click(function() {
var kp = 0;
if ($('#kp').is(':checked')) {
kp = '1';
} else {
kp = '0';
}
$("#ajax").html('')
if( $("#master-add").val() == "" || $("#slave-add").val() == "" || $("#interface-add").val() == "" ||
$("#vrrp-ip-add").val() == "") {
toastr.warning('Please fill in all fields')
} else if(! $("#vrrp-ip-add").val().match(ipformat)) {
toastr.warning('Please enter IP in "VRRP IP" field')
} else {
$("#ajax").html(wait_mess);
$.ajax( {
url: "options.py",
data: {
masteradd: $('#master-add').val(),
slaveadd: $('#slave-add').val(),
interfaceadd: $("#interface-add").val(),
vrrpipadd: $('#vrrp-ip-add').val(),
kp: kp,
token: $('#token').val()
},
type: "POST",
success: function( data ) {
data = data.replace(/\s+/g,' ');
if (data.indexOf('error:') != '-1') {
toastr.clear();
toastr.error(data);
} else if (data.indexOf('success') != '-1'){
toastr.clear();
toastr.success(data);
} else {
toastr.clear();
toastr.info(data);
}
}
} );
}
});
var server_creating = $( "#server_creating" ).dialog({
autoOpen: false,
width: 574,
modal: true,
title: "Creating a new HA cluster",
buttons: {
Close: function() {
$( this ).dialog( "close" );
cleanProvisioningProccess('#server_creating ul li', '#created-mess');
$('#wait_mess').show();
}
}
});
$('#create').click(function() {
var hap = 0;
var nginx = 0;
var syn_flood = 0;
if ($('#hap').is(':checked')) {
hap = '1';
}
if ($('#nginx').is(':checked')) {
nginx = '1';
}
if ($('#syn_flood').is(':checked')) {
syn_flood = '1';
}
$("#ajax").html('')
if( $("#master").val() == "" || $("#slave").val() == "" || $("#interface").val() == "" ||
$("#vrrp-ip").val() == "") {
toastr.warning('Please fill in all fields');
} else if(! $("#vrrp-ip").val().match(ipformat)) {
toastr.warning('Please enter IP in "VRRP IP" field');
} else if ($("#master").val() == $("#slave").val() ){
toastr.warning('Master and slave must be diff servers');
} else {
$("#wait-mess").html(wait_mess);
server_creating.dialog('open');
create_master_keepalived(hap, nginx, syn_flood);
create_slave_keepalived(hap, nginx, syn_flood);
if (hap == '1') {
$('#haproxy_installing_div').show();
}
if (nginx == '1') {
$('#nginx_installing_div').show();
}
}
});
});
function create_master_keepalived(hap, nginx, syn_flood) {
if (hap == '0' && nginx == '0') {
var progress_value = '50';
} else if (hap == '1' || nginx == '0') {
var progress_value = '43';
} else if (hap == '1' && nginx == '1') {
var progress_value = '50';
}
$.ajax( {
url: "options.py",
data: {
master: $('#master').val(),
slave: $('#slave').val(),
interface: $("#interface").val(),
vrrpip: $('#vrrp-ip').val(),
hap: hap,
nginx: nginx,
syn_flood: syn_flood,
token: $('#token').val()
},
type: "POST",
success: function( data ) {
data = data.replace(/\s+/g,' ');
if (data.indexOf('error:') != '-1' || data.indexOf('FAILED') != '-1') {
showProvisioningError(data, '#creating-master', '#wait-mess', '#creating-error');
} else if (data == '' ){
showProvisioningWarning(step_id, 'master Keepalived');
} else if (data.indexOf('success') != '-1' ){
showProvisioningProccess('<p>'+data+'</p>', '#creating-master', progress_value, '#creating-progress');
if (hap === '1') {
create_keep_alived_hap(nginx, 'master');
}
if (hap == '0' && nginx == '1') {
create_keep_alived_nginx('master');
}
} else {
toastr.clear();
toastr.info(data);
}
}
} );
}
function create_slave_keepalived(hap, nginx, syn_flood) {
if (hap == '0' && nginx == '0') {
var progress_value = '100';
} else if (hap == '1' || nginx == '0') {
var progress_value = '67';
} else if (hap == '1' && nginx == '1') {
var progress_value = '50';
}
$.ajax( {
url: "options.py",
data: {
master_slave: $('#master').val(),
slave: $('#slave').val(),
interface: $("#interface").val(),
vrrpip: $('#vrrp-ip').val(),
hap: hap,
nginx: nginx,
syn_flood: syn_flood,
token: $('#token').val()
},
type: "POST",
success: function( data ) {
data = data.replace(/\s+/g,' ');
if (data.indexOf('error:') != '-1' || data.indexOf('FAILED') != '-1') {
showProvisioningError(data, '#creating-slave', '#wait-mess', '#creating-error');
} else if (data == '' ){
showProvisioningWarning(step_id, 'slave Keepalived');
} else if (data.indexOf('success') != '-1' ){
showProvisioningProccess('<p>'+data+'</p>', '#creating-slave', progress_value, '#creating-progress');
} else {
toastr.clear();
toastr.info(data);
}
if (hap === '1') {
create_keep_alived_hap(nginx, 'slave');
}
if (hap == '0' && nginx == '1') {
create_keep_alived_nginx('slave');
}
}
} );
}
function create_keep_alived_hap(nginx, server) {
if (nginx == '0') {
var progress_value = '100';
} else if (nginx == '1') {
var progress_value = '75';
}
if (server === 'master') {
var step_id = '#creating-haproxy-master';
var install_step = 'master Haproxy';
} else {
var step_id = '#creating-haproxy-slave';
var install_step = 'slave Haproxy';
}
$(step_id).addClass('proccessing');
$.ajax( {
url: "options.py",
data: {
master_slave_hap: $('#master').val(),
slave: $('#slave').val(),
server: server,
token: $('#token').val()
},
type: "POST",
success: function( data ) {
data = data.replace(/\s+/g,' ');
if (data.indexOf('error:') != '-1' || data.indexOf('FAILED') != '-1') {
showProvisioningError(data, step_id, '#wait-mess', '#creating-error');
} else if (data == '' ){
showProvisioningWarning(step_id, install_step);
} else if (data.indexOf('success') != '-1' ){
showProvisioningProccess('<br>'+data, step_id, progress_value, '#creating-progress');
} else {
toastr.clear();
toastr.info(data);
}
if (nginx == '1') {
create_keep_alived_nginx(server)
}
}
} );
}
function create_keep_alived_nginx(server) {
if (server === 'master') {
var step_id = '#creating-nginx-master';
var install_step = 'master Nginx';
} else {
var step_id = '#creating-nginx-slave';
var install_step = 'slave Nginx';
}
$(step_id).addClass('proccessing');
$.ajax( {
url: "options.py",
data: {
master_slave_nginx: $('#master').val(),
slave: $('#slave').val(),
server: server,
token: $('#token').val()
},
type: "POST",
success: function( data ) {
data = data.replace(/\s+/g,' ');
if (data.indexOf('error:') != '-1' || data.indexOf('FAILED') != '-1') {
showProvisioningError(data, step_id, '#wait-mess', '#creating-error');
} else if (data == '' ){
showProvisioningWarning(step_id, install_step);
} else if (data.indexOf('success') != '-1' ){
showProvisioningProccess('<br>'+data, step_id, '100', '#creating-progress');
} else {
toastr.clear();
toastr.info(data);
}
}
} );
}
function showProvisioningError(data, step_id, wait_mess, error_id) {
$(wait_mess).hide();
$(error_id).append(data);
$(error_id).show();
$(step_id).removeClass('proccessing');
$(step_id).addClass('processing_error');
$.getScript("/inc/fontawesome.min.js");
}
function showProvisioningWarning(step_id, install_step) {
$('#creating-warning').append('<p>Something went wrong with installation on ' + install_step + ', check logs</p>');
$('#creating-warning').show();
$(step_id).removeClass('proccessing');
$(step_id).addClass('processing_warning');
$('#wait_mess').hide();
$.getScript("/inc/fontawesome.min.js");
}
function cleanProvisioningProccess(div_id, success_div) {
$(success_div).empty();
$(success_div).hide();
$('#creating-error').empty();
$('#creating-error').hide();
$('#creating-warning').empty();
$('#creating-warning').hide();
$('#creating-progress').css('width', '0%');
$('#haproxy_installing_div').hide();
$('#nginx_installing_div').hide();
$(div_id).each(function () {
$(this).removeClass('proccessing_done');
$(this).removeClass('processing_error');
$(this).removeClass('processing_warning');
$(this).removeClass('proccessing');
});
$('#creating-master').addClass('proccessing');
$('#creating-slave').addClass('proccessing');
$.getScript("/inc/fontawesome.min.js");
}
function showProvisioningProccess(data, step_id, progress_value, progress_id) {
$(step_id).addClass('proccessing_done');
$(step_id).removeClass('proccessing');
$('#created-mess').show();
$('#created-mess').append(data);
$(progress_id).css('width', progress_value+'%');
if (progress_value === '100')
$("#wait-mess").hide();
$.getScript("/inc/fontawesome.min.js");
}

7
inc/provisioning.css
View File

@ -7,7 +7,7 @@
font-family: "Font Awesome 5 Solid";
content: "\f35a";
}
.processing .fa-arrow-alt-circle-right {
.proccessing .fa-arrow-alt-circle-right {
padding-right: 10px !important;
margin-bottom: -1px !important;
}
@ -16,7 +16,8 @@
margin-bottom: -1px !important;
}
.processing_error { color: red;}
.processing_error::before {
.processing_warning { color: #efba22;}
.processing_error::before, .processing_warning::before {
display: none;
font-family: "Font Awesome 5 Solid";
content: "\f06a";
@ -88,4 +89,4 @@
}
.first-collumn {
width: 15%;
}
}

99
inc/users.js
View File

@ -58,103 +58,6 @@ $( function() {
autoFocus: true,
minLength: -1
});
var ipformat = /^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/;
$('#create').click(function() {
var hap = 0;
var nginx = 0;
var syn_flood = 0;
if ($('#hap').is(':checked')) {
hap = '1';
}
if ($('#nginx').is(':checked')) {
nginx = '1';
}
if ($('#syn_flood').is(':checked')) {
syn_flood = '1';
}
$("#ajax").html('')
if( $("#master").val() == "" || $("#slave").val() == "" || $("#interface").val() == "" ||
$("#vrrp-ip").val() == "") {
toastr.warning('Please fill in all fields');
} else if(! $("#vrrp-ip").val().match(ipformat)) {
toastr.warning('Please enter IP in "VRRP IP" field');
} else if ($("#master").val() == $("#slave").val() ){
toastr.warning('Master and slave must be diff servers');
} else {
$("#ajax").html(wait_mess);
$.ajax( {
url: "options.py",
data: {
master: $('#master').val(),
slave: $('#slave').val(),
interface: $("#interface").val(),
vrrpip: $('#vrrp-ip').val(),
hap: hap,
nginx: nginx,
syn_flood: syn_flood,
token: $('#token').val()
},
type: "POST",
success: function( data ) {
data = data.replace(/\s+/g,' ');
if (data.indexOf('error:') != '-1' || data.indexOf('alert') != '-1' || data.indexOf('FAILED') != '-1') {
toastr.error(data);
} else if (data.indexOf('info') != '-1' ){
toastr.clear();
toastr.info(data);
} else if (data.indexOf('success') != '-1' ){
toastr.clear();
toastr.success(data);
} else {
toastr.clear();
toastr.info(data);
}
}
} );
}
});
$('#add-vrrp').click(function() {
var kp = 0;
if ($('#kp').is(':checked')) {
kp = '1';
} else {
kp = '0';
}
$("#ajax").html('')
if( $("#master-add").val() == "" || $("#slave-add").val() == "" || $("#interface-add").val() == "" ||
$("#vrrp-ip-add").val() == "") {
toastr.warning('Please fill in all fields')
} else if(! $("#vrrp-ip-add").val().match(ipformat)) {
toastr.warning('Please enter IP in "VRRP IP" field')
} else {
$("#ajax").html(wait_mess);
$.ajax( {
url: "options.py",
data: {
masteradd: $('#master-add').val(),
slaveadd: $('#slave-add').val(),
interfaceadd: $("#interface-add").val(),
vrrpipadd: $('#vrrp-ip-add').val(),
kp: kp,
token: $('#token').val()
},
type: "POST",
success: function( data ) {
data = data.replace(/\s+/g,' ');
if (data.indexOf('error:') != '-1') {
toastr.clear();
toastr.error(data);
} else if (data.indexOf('success') != '-1'){
toastr.clear();
toastr.success(data);
} else {
toastr.clear();
toastr.info(data);
}
}
} );
}
});
$('#install').click(function() {
$("#ajax").html('')
var syn_flood = 0;
@ -206,7 +109,7 @@ $( function() {
},
type: "POST",
success: function( data ) {
data = data.replace(/\s+/g,' ');
data = data.replace(/\s+/g,' ');
$("#ajax").html('')
if (data.indexOf('error:') != '-1' || data.indexOf('FAILED') != '-1') {
toastr.clear();