From 59b40578d813367f98cd42cff380ec4da2e25386 Mon Sep 17 00:00:00 2001 From: fit2bot <68588906+fit2bot@users.noreply.github.com> Date: Tue, 9 Sep 2025 14:26:42 +0800 Subject: [PATCH] fix: adhoc SQL Server 2008 (#15984) * fix: Resolve the issue of errors occurring during automated execution with SQL Server 2008 * fix: adhoc SQL Server 2008 * perf: add todo information --------- Co-authored-by: halo --- .../change_secret/database/sqlserver/main.yml | 20 +- .../database/sqlserver/main.yml | 4 +- .../push_account/database/sqlserver/main.yml | 20 +- .../database/sqlserver/main.yml | 4 +- .../database/sqlserver/main.yml | 4 +- .../ping/database/sqlserver/main.yml | 4 +- apps/libs/ansible/modules/mssql_script.py | 401 ++++++++++++++++++ apps/ops/ansible/inventory.py | 14 +- apps/ops/models/job.py | 9 +- 9 files changed, 460 insertions(+), 20 deletions(-) create mode 100644 apps/libs/ansible/modules/mssql_script.py diff --git a/apps/accounts/automations/change_secret/database/sqlserver/main.yml b/apps/accounts/automations/change_secret/database/sqlserver/main.yml index 4e8cd5d52..90495be45 100644 --- a/apps/accounts/automations/change_secret/database/sqlserver/main.yml +++ b/apps/accounts/automations/change_secret/database/sqlserver/main.yml @@ -5,12 +5,14 @@ tasks: - name: Test SQLServer connection - community.general.mssql_script: + mssql_script: login_user: "{{ jms_account.username }}" login_password: "{{ jms_account.secret }}" login_host: "{{ jms_asset.address }}" login_port: "{{ jms_asset.port }}" name: '{{ jms_asset.spec_info.db_name }}' + encryption: "{{ jms_asset.encryption | default(None) }}" + tds_version: "{{ jms_asset.tds_version | default(None) }}" script: | SELECT @@version register: db_info @@ -23,45 +25,53 @@ var: info - name: Check whether SQLServer User exist - community.general.mssql_script: + mssql_script: login_user: "{{ jms_account.username }}" login_password: "{{ jms_account.secret }}" login_host: "{{ jms_asset.address }}" login_port: "{{ jms_asset.port }}" name: '{{ jms_asset.spec_info.db_name }}' + encryption: "{{ jms_asset.encryption | default(None) }}" + tds_version: "{{ jms_asset.tds_version | default(None) }}" script: "SELECT 1 from sys.sql_logins WHERE name='{{ account.username }}';" when: db_info is succeeded register: user_exist - name: Change SQLServer password - community.general.mssql_script: + mssql_script: login_user: "{{ jms_account.username }}" login_password: "{{ jms_account.secret }}" login_host: "{{ jms_asset.address }}" login_port: "{{ jms_asset.port }}" name: '{{ jms_asset.spec_info.db_name }}' + encryption: "{{ jms_asset.encryption | default(None) }}" + tds_version: "{{ jms_asset.tds_version | default(None) }}" script: "ALTER LOGIN {{ account.username }} WITH PASSWORD = '{{ account.secret }}', DEFAULT_DATABASE = {{ jms_asset.spec_info.db_name }}; select @@version" ignore_errors: true when: user_exist.query_results[0] | length != 0 - name: Add SQLServer user - community.general.mssql_script: + mssql_script: login_user: "{{ jms_account.username }}" login_password: "{{ jms_account.secret }}" login_host: "{{ jms_asset.address }}" login_port: "{{ jms_asset.port }}" name: '{{ jms_asset.spec_info.db_name }}' + encryption: "{{ jms_asset.encryption | default(None) }}" + tds_version: "{{ jms_asset.tds_version | default(None) }}" script: "CREATE LOGIN {{ account.username }} WITH PASSWORD = '{{ account.secret }}', DEFAULT_DATABASE = {{ jms_asset.spec_info.db_name }}; CREATE USER {{ account.username }} FOR LOGIN {{ account.username }}; select @@version" ignore_errors: true when: user_exist.query_results[0] | length == 0 - name: Verify password - community.general.mssql_script: + mssql_script: login_user: "{{ account.username }}" login_password: "{{ account.secret }}" login_host: "{{ jms_asset.address }}" login_port: "{{ jms_asset.port }}" name: '{{ jms_asset.spec_info.db_name }}' + encryption: "{{ jms_asset.encryption | default(None) }}" + tds_version: "{{ jms_asset.tds_version | default(None) }}" script: | SELECT @@version when: check_conn_after_change diff --git a/apps/accounts/automations/gather_account/database/sqlserver/main.yml b/apps/accounts/automations/gather_account/database/sqlserver/main.yml index ce8a20e0d..2ee29f672 100644 --- a/apps/accounts/automations/gather_account/database/sqlserver/main.yml +++ b/apps/accounts/automations/gather_account/database/sqlserver/main.yml @@ -5,12 +5,14 @@ tasks: - name: Test SQLServer connection - community.general.mssql_script: + mssql_script: login_user: "{{ jms_account.username }}" login_password: "{{ jms_account.secret }}" login_host: "{{ jms_asset.address }}" login_port: "{{ jms_asset.port }}" name: '{{ jms_asset.spec_info.db_name }}' + encryption: "{{ jms_asset.encryption | default(None) }}" + tds_version: "{{ jms_asset.tds_version | default(None) }}" script: | SELECT l.name, diff --git a/apps/accounts/automations/push_account/database/sqlserver/main.yml b/apps/accounts/automations/push_account/database/sqlserver/main.yml index bb66c559a..b3c5a8ab4 100644 --- a/apps/accounts/automations/push_account/database/sqlserver/main.yml +++ b/apps/accounts/automations/push_account/database/sqlserver/main.yml @@ -5,12 +5,14 @@ tasks: - name: Test SQLServer connection - community.general.mssql_script: + mssql_script: login_user: "{{ jms_account.username }}" login_password: "{{ jms_account.secret }}" login_host: "{{ jms_asset.address }}" login_port: "{{ jms_asset.port }}" name: '{{ jms_asset.spec_info.db_name }}' + encryption: "{{ jms_asset.encryption | default(None) }}" + tds_version: "{{ jms_asset.tds_version | default(None) }}" script: | SELECT @@version register: db_info @@ -23,47 +25,55 @@ var: info - name: Check whether SQLServer User exist - community.general.mssql_script: + mssql_script: login_user: "{{ jms_account.username }}" login_password: "{{ jms_account.secret }}" login_host: "{{ jms_asset.address }}" login_port: "{{ jms_asset.port }}" name: '{{ jms_asset.spec_info.db_name }}' + encryption: "{{ jms_asset.encryption | default(None) }}" + tds_version: "{{ jms_asset.tds_version | default(None) }}" script: "SELECT 1 from sys.sql_logins WHERE name='{{ account.username }}';" when: db_info is succeeded register: user_exist - name: Change SQLServer password - community.general.mssql_script: + mssql_script: login_user: "{{ jms_account.username }}" login_password: "{{ jms_account.secret }}" login_host: "{{ jms_asset.address }}" login_port: "{{ jms_asset.port }}" name: '{{ jms_asset.spec_info.db_name }}' + encryption: "{{ jms_asset.encryption | default(None) }}" + tds_version: "{{ jms_asset.tds_version | default(None) }}" script: "ALTER LOGIN {{ account.username }} WITH PASSWORD = '{{ account.secret }}', DEFAULT_DATABASE = {{ jms_asset.spec_info.db_name }}; select @@version" ignore_errors: true when: user_exist.query_results[0] | length != 0 register: change_info - name: Add SQLServer user - community.general.mssql_script: + mssql_script: login_user: "{{ jms_account.username }}" login_password: "{{ jms_account.secret }}" login_host: "{{ jms_asset.address }}" login_port: "{{ jms_asset.port }}" name: '{{ jms_asset.spec_info.db_name }}' + encryption: "{{ jms_asset.encryption | default(None) }}" + tds_version: "{{ jms_asset.tds_version | default(None) }}" script: "CREATE LOGIN [{{ account.username }}] WITH PASSWORD = '{{ account.secret }}'; CREATE USER [{{ account.username }}] FOR LOGIN [{{ account.username }}]; select @@version" ignore_errors: true when: user_exist.query_results[0] | length == 0 register: change_info - name: Verify password - community.general.mssql_script: + mssql_script: login_user: "{{ account.username }}" login_password: "{{ account.secret }}" login_host: "{{ jms_asset.address }}" login_port: "{{ jms_asset.port }}" name: '{{ jms_asset.spec_info.db_name }}' + encryption: "{{ jms_asset.encryption | default(None) }}" + tds_version: "{{ jms_asset.tds_version | default(None) }}" script: | SELECT @@version when: check_conn_after_change diff --git a/apps/accounts/automations/remove_account/database/sqlserver/main.yml b/apps/accounts/automations/remove_account/database/sqlserver/main.yml index de68021d2..aa074633d 100644 --- a/apps/accounts/automations/remove_account/database/sqlserver/main.yml +++ b/apps/accounts/automations/remove_account/database/sqlserver/main.yml @@ -5,11 +5,13 @@ tasks: - name: "Remove account" - community.general.mssql_script: + mssql_script: login_user: "{{ jms_account.username }}" login_password: "{{ jms_account.secret }}" login_host: "{{ jms_asset.address }}" login_port: "{{ jms_asset.port }}" name: "{{ jms_asset.spec_info.db_name }}" + encryption: "{{ jms_asset.encryption | default(None) }}" + tds_version: "{{ jms_asset.tds_version | default(None) }}" script: "DROP LOGIN {{ account.username }}; select @@version" diff --git a/apps/accounts/automations/verify_account/database/sqlserver/main.yml b/apps/accounts/automations/verify_account/database/sqlserver/main.yml index 0a0509656..83b1835f3 100644 --- a/apps/accounts/automations/verify_account/database/sqlserver/main.yml +++ b/apps/accounts/automations/verify_account/database/sqlserver/main.yml @@ -5,11 +5,13 @@ tasks: - name: Verify account - community.general.mssql_script: + mssql_script: login_user: "{{ account.username }}" login_password: "{{ account.secret }}" login_host: "{{ jms_asset.address }}" login_port: "{{ jms_asset.port }}" name: '{{ jms_asset.spec_info.db_name }}' + encryption: "{{ jms_asset.encryption | default(None) }}" + tds_version: "{{ jms_asset.tds_version | default(None) }}" script: | SELECT @@version diff --git a/apps/assets/automations/ping/database/sqlserver/main.yml b/apps/assets/automations/ping/database/sqlserver/main.yml index a1a1939b2..080cbd785 100644 --- a/apps/assets/automations/ping/database/sqlserver/main.yml +++ b/apps/assets/automations/ping/database/sqlserver/main.yml @@ -6,11 +6,13 @@ tasks: - name: Test SQLServer connection - community.general.mssql_script: + mssql_script: login_user: "{{ jms_account.username }}" login_password: "{{ jms_account.secret }}" login_host: "{{ jms_asset.address }}" login_port: "{{ jms_asset.port }}" name: '{{ jms_asset.spec_info.db_name }}' + encryption: "{{ jms_asset.encryption | default(None) }}" + tds_version: "{{ jms_asset.tds_version | default(None) }}" script: | SELECT @@version diff --git a/apps/libs/ansible/modules/mssql_script.py b/apps/libs/ansible/modules/mssql_script.py new file mode 100644 index 000000000..f8984aa4d --- /dev/null +++ b/apps/libs/ansible/modules/mssql_script.py @@ -0,0 +1,401 @@ +#!/usr/bin/python + +# Modified from ansible_collections.community.general.mssql_script + +from __future__ import (absolute_import, division, print_function) + +__metaclass__ = type + +DOCUMENTATION = r""" +module: mssql_script + +short_description: Execute SQL scripts on a MSSQL database + +version_added: "1.0.0" + +description: + - Execute SQL scripts on a MSSQL database. +extends_documentation_fragment: + - community.general.attributes + +attributes: + check_mode: + support: partial + details: + - The script is not be executed in check mode. + diff_mode: + support: none + +options: + name: + description: Database to run script against. + aliases: [db] + default: '' + type: str + login_user: + description: The username used to authenticate with. + type: str + login_password: + description: The password used to authenticate with. + type: str + login_host: + description: Host running the database. + type: str + required: true + login_port: + description: Port of the MSSQL server. Requires O(login_host) be defined as well. + default: 1433 + type: int + script: + description: + - The SQL script to be executed. + - Script can contain multiple SQL statements. Multiple Batches can be separated by V(GO) command. + - Each batch must return at least one result set. + required: true + type: str + transaction: + description: + - If transactional mode is requested, start a transaction and commit the change only if the script succeed. Otherwise, + rollback the transaction. + - If transactional mode is not requested (default), automatically commit the change. + type: bool + default: false + version_added: 8.4.0 + output: + description: + - With V(default) each row is returned as a list of values. See RV(query_results). + - Output format V(dict) returns dictionary with the column names as keys. See RV(query_results_dict). + - V(dict) requires named columns to be returned by each query otherwise an error is thrown. + choices: ["dict", "default"] + default: 'default' + type: str + params: + description: |- + Parameters passed to the script as SQL parameters. + (Query V('SELECT %(name\)s"') with V(example: '{"name": "John Doe"}).)'. + type: dict +notes: + - Requires the pymssql Python package on the remote host. For Ubuntu, this is as easy as C(pip install pymssql) (See M(ansible.builtin.pip)). +requirements: + - pymssql + +author: + - Kris Budde (@kbudde) +""" + +EXAMPLES = r""" +- name: Check DB connection + community.general.mssql_script: + login_user: "{{ mssql_login_user }}" + login_password: "{{ mssql_login_password }}" + login_host: "{{ mssql_host }}" + login_port: "{{ mssql_port }}" + db: master + script: "SELECT 1" + +- name: Query with parameter + community.general.mssql_script: + login_user: "{{ mssql_login_user }}" + login_password: "{{ mssql_login_password }}" + login_host: "{{ mssql_host }}" + login_port: "{{ mssql_port }}" + script: | + SELECT name, state_desc FROM sys.databases WHERE name = %(dbname)s + params: + dbname: msdb + register: result_params +- assert: + that: + - result_params.query_results[0][0][0][0] == 'msdb' + - result_params.query_results[0][0][0][1] == 'ONLINE' + +- name: Query within a transaction + community.general.mssql_script: + login_user: "{{ mssql_login_user }}" + login_password: "{{ mssql_login_password }}" + login_host: "{{ mssql_host }}" + login_port: "{{ mssql_port }}" + script: | + UPDATE sys.SomeTable SET desc = 'some_table_desc' WHERE name = %(dbname)s + UPDATE sys.AnotherTable SET desc = 'another_table_desc' WHERE name = %(dbname)s + transaction: true + params: + dbname: msdb + +- name: two batches with default output + community.general.mssql_script: + login_user: "{{ mssql_login_user }}" + login_password: "{{ mssql_login_password }}" + login_host: "{{ mssql_host }}" + login_port: "{{ mssql_port }}" + script: | + SELECT 'Batch 0 - Select 0' + SELECT 'Batch 0 - Select 1' + GO + SELECT 'Batch 1 - Select 0' + register: result_batches +- assert: + that: + - result_batches.query_results | length == 2 # two batch results + - result_batches.query_results[0] | length == 2 # two selects in first batch + - result_batches.query_results[0][0] | length == 1 # one row in first select + - result_batches.query_results[0][0][0] | length == 1 # one column in first row + - result_batches.query_results[0][0][0][0] == 'Batch 0 - Select 0' # each row contains a list of values. + +- name: two batches with dict output + community.general.mssql_script: + login_user: "{{ mssql_login_user }}" + login_password: "{{ mssql_login_password }}" + login_host: "{{ mssql_host }}" + login_port: "{{ mssql_port }}" + output: dict + script: | + SELECT 'Batch 0 - Select 0' as b0s0 + SELECT 'Batch 0 - Select 1' as b0s1 + GO + SELECT 'Batch 1 - Select 0' as b1s0 + register: result_batches_dict +- assert: + that: + - result_batches_dict.query_results_dict | length == 2 # two batch results + - result_batches_dict.query_results_dict[0] | length == 2 # two selects in first batch + - result_batches_dict.query_results_dict[0][0] | length == 1 # one row in first select + - result_batches_dict.query_results_dict[0][0][0]['b0s0'] == 'Batch 0 - Select 0' # column 'b0s0' of first row +""" + +RETURN = r""" +query_results: + description: List of batches (queries separated by V(GO) keyword). + type: list + elements: list + returned: success and O(output=default) + sample: + [ + [ + [ + [ + "Batch 0 - Select 0" + ] + ], + [ + [ + "Batch 0 - Select 1" + ] + ] + ], + [ + [ + [ + "Batch 1 - Select 0" + ] + ] + ] + ] + contains: + queries: + description: + - List of result sets of each query. + - If a query returns no results, the results of this and all the following queries are not included in the output. + - Use the V(GO) keyword in O(script) to separate queries. + type: list + elements: list + contains: + rows: + description: List of rows returned by query. + type: list + elements: list + contains: + column_value: + description: + - List of column values. + - Any non-standard JSON type is converted to string. + type: list + example: ["Batch 0 - Select 0"] + returned: success, if output is default +query_results_dict: + description: List of batches (queries separated by V(GO) keyword). + type: list + elements: list + returned: success and O(output=dict) + sample: + [ + [ + [ + [ + "Batch 0 - Select 0" + ] + ], + [ + [ + "Batch 0 - Select 1" + ] + ] + ], + [ + [ + [ + "Batch 1 - Select 0" + ] + ] + ] + ] + contains: + queries: + description: + - List of result sets of each query. + - If a query returns no results, the results of this and all the following queries are not included in the output. + Use V(GO) keyword to separate queries. + type: list + elements: list + contains: + rows: + description: List of rows returned by query. + type: list + elements: list + contains: + column_dict: + description: + - Dictionary of column names and values. + - Any non-standard JSON type is converted to string. + type: dict + example: {"col_name": "Batch 0 - Select 0"} + returned: success, if output is dict +""" + +import pymssql +from ansible.module_utils.basic import AnsibleModule +import json + + +def clean_output(o): + return str(o) + + +def main(): + module_args = dict( + name=dict(aliases=['db'], default=''), + login_user=dict(type='str', required=False, default=None), + login_password=dict(no_log=True), + login_host=dict(required=True), + login_port=dict(type='int', default=1433), + script=dict(required=True), + output=dict(default='default', choices=['dict', 'default']), + params=dict(type='dict'), + transaction=dict(type='bool', default=True), + tds_version=dict(type='str', required=False, default=None), + encryption=dict(type='str', required=False, default=None) + ) + + result = dict( + changed=False, + ) + + module = AnsibleModule( + argument_spec=module_args, + supports_check_mode=True + ) + + db = module.params['name'] + login_user = module.params['login_user'] + login_password = module.params['login_password'] + login_host = module.params['login_host'] + login_port = module.params['login_port'] + script = module.params['script'] + output = module.params['output'] + sql_params = module.params['params'] + transaction = module.params['transaction'] + # TODO 待 ansible 官方支持这两个参数 + tds_version = module.params['tds_version'] or None + encryption = module.params['encryption'] or None + + login_querystring = login_host + if login_port != 1433: + login_querystring = "%s:%s" % (login_host, login_port) + + if login_user is not None and login_password is None: + module.fail_json( + msg="when supplying login_user argument, login_password must also be provided") + + try: + conn = pymssql.connect( + user=login_user, password=login_password, host=login_querystring, + database=db, encryption=encryption, tds_version=tds_version) + cursor = conn.cursor() + except Exception as e: + if "Unknown database" in str(e): + errno, errstr = e.args + module.fail_json(msg="ERROR: %s %s" % (errno, errstr)) + else: + module.fail_json( + msg="unable to connect, check login_user and login_password are correct, or alternatively check your " + "@sysconfdir@/freetds.conf / ${HOME}/.freetds.conf") + + # If transactional mode is requested, start a transaction + conn.autocommit(not transaction) + + query_results_key = 'query_results' + if output == 'dict': + cursor = conn.cursor(as_dict=True) + query_results_key = 'query_results_dict' + + # Process the script into batches + queries = [] + current_batch = [] + for statement in script.splitlines(True): + # Ignore the Byte Order Mark, if found + if statement.strip() == '\uFEFF': + continue + + # Assume each 'GO' is on its own line but may have leading/trailing whitespace + # and be of mixed-case + if statement.strip().upper() != 'GO': + current_batch.append(statement) + else: + queries.append(''.join(current_batch)) + current_batch = [] + if len(current_batch) > 0: + queries.append(''.join(current_batch)) + + result['changed'] = True + if module.check_mode: + module.exit_json(**result) + + query_results = [] + for query in queries: + # Catch and exit on any bad query errors + try: + cursor.execute(query, sql_params) + qry_result = [] + rows = cursor.fetchall() + while rows: + qry_result.append(rows) + rows = cursor.fetchall() + query_results.append(qry_result) + except Exception as e: + # We know we executed the statement so this error just means we have no resultset + # which is ok (eg UPDATE/INSERT) + if ( + type(e).__name__ == 'OperationalError' and + str(e) == 'Statement not executed or executed statement has no resultset' + ): + query_results.append([]) + else: + # Rollback transaction before failing the module in case of error + if transaction: + conn.rollback() + error_msg = '%s: %s' % (type(e).__name__, str(e)) + module.fail_json(msg="query failed", query=query, error=error_msg, **result) + + # Commit transaction before exiting the module in case of no error + if transaction: + conn.commit() + + # ensure that the result is json serializable + qry_results = json.loads(json.dumps(query_results, default=clean_output)) + + result[query_results_key] = qry_results + module.exit_json(**result) + + +if __name__ == '__main__': + main() diff --git a/apps/ops/ansible/inventory.py b/apps/ops/ansible/inventory.py index 35085aa1f..9e8c1e97b 100644 --- a/apps/ops/ansible/inventory.py +++ b/apps/ops/ansible/inventory.py @@ -112,12 +112,21 @@ class JMSInventory: @staticmethod def make_protocol_setting_vars(host, protocols): - # 针对 ssh 协议的特殊处理 + # 针对 ssh sqlserver 协议的特殊处理 for p in protocols: if p.name == 'ssh': if hasattr(p, 'setting'): setting = getattr(p, 'setting') - host['old_ssh_version'] = setting.get('old_ssh_version', False) + host['jms_asset']['old_ssh_version'] = setting.get('old_ssh_version', False) + if p.name == 'sqlserver': + if hasattr(p, 'setting'): + setting = getattr(p, 'setting') + encryption = setting.get('encrypt', True) + version = setting.get('version', ">=2014") + if version == '<2014': + host['jms_asset']['tds_version'] = '7.0' + if not encryption: + host['jms_asset']['encryption'] = 'off' def make_account_vars(self, host, asset, account, automation, protocol, platform, gateway, path_dir, ansible_config): @@ -226,7 +235,6 @@ class JMSInventory: }) self.make_protocol_setting_vars(host, protocols) - protocols = host['jms_asset']['protocols'] host['jms_asset'].update({f"{p['name']}_port": p['port'] for p in protocols}) if host['jms_account'] and tp == 'oracle': diff --git a/apps/ops/models/job.py b/apps/ops/models/job.py index d3b3336cb..71313d4ee 100644 --- a/apps/ops/models/job.py +++ b/apps/ops/models/job.py @@ -1,11 +1,11 @@ import json import logging import os +import sys import uuid from collections import defaultdict from datetime import timedelta, datetime -import sys from celery import current_task from django.conf import settings from django.db import models @@ -271,7 +271,7 @@ class JobExecution(JMSOrgBaseModel): db_module_name_map = { 'mysql': 'community.mysql.mysql_query', 'postgresql': 'community.postgresql.postgresql_query', - 'sqlserver': 'community.general.mssql_script', + 'sqlserver': 'mssql_script', } extra_query_token_map = { 'sqlserver': 'script' @@ -292,7 +292,10 @@ class JobExecution(JMSOrgBaseModel): "login_user={{login_user}} " \ "login_password={{login_password}} " \ "login_port={{login_port}} " \ - "%s={{login_db}}" % login_db_token + "%s={{login_db}} " % login_db_token + if module == 'mssql_script': + login_args += "encryption={{jms_asset.encryption | default(None) }} " \ + "tds_version={{jms_asset.tds_version | default(None) }} " shell = "{} {}=\"{}\" ".format(login_args, query_token, self.current_job.args) return module, shell