diff --git a/build/builder/build-external.py b/build/builder/build-external.py index a4c3135..d14e793 100644 --- a/build/builder/build-external.py +++ b/build/builder/build-external.py @@ -382,6 +382,7 @@ class BuilderLinux(BuilderBase): self.LIBUV_PATH_SRC = os.path.join(self.PATH_TMP, 'libuv-{}'.format(env.ver_libuv)) self.MBEDTLS_PATH_SRC = os.path.join(self.PATH_TMP, 'mbedtls-mbedtls-{}'.format(env.ver_mbedtls)) self.LIBSSH_PATH_SRC = os.path.join(self.PATH_TMP, 'libssh-{}'.format(env.ver_libssh)) + self.ZLIB_PATH_SRC = os.path.join(self.PATH_TMP, 'zlib-{}'.format(env.ver_zlib)) self.JSONCPP_PATH_SRC = os.path.join(PATH_EXTERNAL, 'jsoncpp') self.MONGOOSE_PATH_SRC = os.path.join(PATH_EXTERNAL, 'mongoose') @@ -564,7 +565,48 @@ class BuilderLinux(BuilderBase): # os.unlink(os.path.join(self.PATH_RELEASE, 'lib', i)) def _build_zlib(self, file_name): - cc.w('skip build zlib again.') + # cc.w('skip build zlib again.') + if not self._download_zlib(file_name): + return + if not os.path.exists(self.ZLIB_PATH_SRC): + os.system('unzip "{}/{}" -d "{}"'.format(PATH_DOWNLOAD, file_name, self.PATH_TMP)) + + cc.n('build zlib...', end='') + out_file = os.path.join(self.PATH_RELEASE, 'lib', 'libz.a') + if os.path.exists(out_file): + cc.w('already exists, skip.') + return + cc.v('') + + # cc.n('fix libssh source code... ', end='') + # s_name = 'libssh-{}'.format(env.ver_libssh) + # # utils.ensure_file_exists(os.path.join(PATH_EXTERNAL, 'fix-external', 'libssh', s_name, 'src', 'session.c')) + # # utils.ensure_file_exists(os.path.join(PATH_EXTERNAL, 'fix-external', 'libssh', s_name, 'src', 'libcrypto.c')) + # utils.ensure_file_exists(os.path.join(PATH_EXTERNAL, 'fix-external', 'libssh', s_name, 'src', 'libcrypto-compat.c')) + # # utils.copy_file(os.path.join(PATH_EXTERNAL, 'fix-external', 'libssh', s_name, 'src'), os.path.join(self.LIBSSH_PATH_SRC, 'src'), 'session.c') + # # utils.copy_file(os.path.join(PATH_EXTERNAL, 'fix-external', 'libssh', s_name, 'src'), os.path.join(self.LIBSSH_PATH_SRC, 'src'), 'libcrypto.c') + # utils.copy_file(os.path.join(PATH_EXTERNAL, 'fix-external', 'libssh', s_name, 'src'), os.path.join(self.LIBSSH_PATH_SRC, 'src'), 'libcrypto-compat.c') + + build_path = os.path.join(self.ZLIB_PATH_SRC, 'build') + + cmake_define = ' -DCMAKE_INSTALL_PREFIX={path_release}' \ + ' ..'.format(path_release=self.PATH_RELEASE) + + old_p = os.getcwd() + try: + utils.cmake(build_path, 'Release', False, cmake_define=cmake_define, cmake_pre_define='CFLAGS="-fPIC"') + os.chdir(build_path) + utils.sys_exec('make install') + except: + pass + os.chdir(old_p) + + utils.ensure_file_exists(out_file) + files = os.listdir(os.path.join(self.PATH_RELEASE, 'lib')) + for i in files: + if i.startswith('libz.so'): + # use os.unlink() because some file should be a link. + os.unlink(os.path.join(self.PATH_RELEASE, 'lib', i)) def fix_output(self): pass diff --git a/build/builder/build-pysrt.py b/build/builder/build-pysrt.py index e979252..965340e 100644 --- a/build/builder/build-pysrt.py +++ b/build/builder/build-pysrt.py @@ -46,7 +46,7 @@ class PYSBase: utils.sys_exec('{} -m pip install -i https://pypi.tuna.tsinghua.edu.cn/simple pip --upgrade'.format(env.py_exec)) pip = self._get_pip() - pypi_modules = ['ldap3', 'mako', 'Pillow', 'psutil', 'pymysql', 'qrcode', 'tornado', 'wheezy.captcha'] + pypi_modules = ['cryptography', 'ldap3', 'mako', 'Pillow', 'psutil', 'pyasn1', 'pymysql', 'qrcode', 'tornado', 'wheezy.captcha'] for p in pypi_modules: cc.n('install {} ...'.format(p)) utils.sys_exec('{} install -i https://pypi.tuna.tsinghua.edu.cn/simple {}'.format(pip, p), direct_output=True) diff --git a/common/libex/src/ex_thread.cpp b/common/libex/src/ex_thread.cpp index 9358a77..395dfba 100644 --- a/common/libex/src/ex_thread.cpp +++ b/common/libex/src/ex_thread.cpp @@ -80,8 +80,10 @@ bool ExThreadBase::stop(void) { return false; } #else - if (pthread_join(m_handle, NULL) != 0) { - return false; + if(m_handle != 0) { + if (pthread_join(m_handle, NULL) != 0) { + return false; + } } #endif diff --git a/dist/client/windows/assist/installer.nsi b/dist/client/windows/assist/installer.nsi index f6fc853..db662a9 100644 Binary files a/dist/client/windows/assist/installer.nsi and b/dist/client/windows/assist/installer.nsi differ diff --git a/server/tp_core/protocol/ssh/CMakeLists.txt b/server/tp_core/protocol/ssh/CMakeLists.txt index 98f736b..4d122e0 100644 --- a/server/tp_core/protocol/ssh/CMakeLists.txt +++ b/server/tp_core/protocol/ssh/CMakeLists.txt @@ -8,8 +8,9 @@ MESSAGE(STATUS "=======================================================") include(../../../../CMakeCfg.txt) -set(CMAKE_CXX_FLAGS "-fPIC") -set(CMAKE_C_FLAGS "-fPIC") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") + aux_source_directory(. DIR_SSH_SRCS) aux_source_directory(../../common DIR_SSH_SRCS) @@ -27,15 +28,18 @@ include_directories( ) include_directories( - ${TP_EXTERNAL_RELEASE_DIR}/include -) -link_directories(${TP_EXTERNAL_RELEASE_DIR}/lib) + ${TP_EXTERNAL_RELEASE_DIR}/include + ) +link_directories( + ${TP_EXTERNAL_RELEASE_DIR}/lib + ${TP_EXTERNAL_RELEASE_DIR}/lib64 + ) add_library(tpssh SHARED ${DIR_SSH_SRCS}) if (OS_LINUX) - target_link_libraries(tpssh ssh ssl crypto mbedx509 mbedtls mbedcrypto dl pthread rt util) + target_link_libraries(tpssh ssh ssl crypto mbedx509 mbedtls mbedcrypto z dl pthread rt util) elseif (OS_MACOS) - target_link_libraries(tpssh ssh ssl crypto mbedx509 mbedtls mbedcrypto dl pthread util) + target_link_libraries(tpssh ssh ssl crypto mbedx509 mbedtls mbedcrypto z dl pthread util) endif() diff --git a/server/tp_core/protocol/ssh/ssh_proxy.cpp b/server/tp_core/protocol/ssh/ssh_proxy.cpp index 3a8f4d4..324642c 100644 --- a/server/tp_core/protocol/ssh/ssh_proxy.cpp +++ b/server/tp_core/protocol/ssh/ssh_proxy.cpp @@ -185,7 +185,10 @@ void SshProxy::_on_stop() { int _timeout_us = 10; ssh_options_set(_session, SSH_OPTIONS_TIMEOUT, &_timeout_us); ssh_connect(_session); + ssh_disconnect(_session); ssh_free(_session); + + ex_sleep_ms(100); } // m_thread_mgr.stop_all(); diff --git a/server/tp_core/protocol/telnet/telnet_recorder.cpp b/server/tp_core/protocol/telnet/telnet_recorder.cpp index 985e896..c1700ee 100644 --- a/server/tp_core/protocol/telnet/telnet_recorder.cpp +++ b/server/tp_core/protocol/telnet/telnet_recorder.cpp @@ -8,7 +8,7 @@ TppTelnetRec::TppTelnetRec() memset(&m_head, 0, sizeof(TS_RECORD_HEADER)); memcpy((ex_u8*)(&m_head.info.magic), TPP_RECORD_MAGIC, sizeof(ex_u32)); - m_head.info.ver = 0x03; + m_head.info.ver = 0x04; m_header_changed = false; m_save_full_header = false; @@ -42,7 +42,7 @@ bool TppTelnetRec::_on_begin(const TPP_CONNECT_INFO* info) bool TppTelnetRec::_on_end() { - // 如果还有剩下未写入的数据，写入文件中。 + // 濡傛灉杩樻湁鍓╀笅鏈啓鍏ョ殑鏁版嵁锛屽啓鍏ユ枃浠朵腑銆 save_record(); if (m_file_info != NULL) @@ -76,13 +76,14 @@ void TppTelnetRec::record(ex_u8 type, const ex_u8* data, size_t size) { pkg.time_ms = (ex_u32)(ex_get_tick_count() - m_start_time); m_head.info.time_ms = pkg.time_ms; + m_header_changed = true; } m_cache.append((ex_u8*)&pkg, sizeof(TS_RECORD_PKG)); m_cache.append(data, size); - m_head.info.packages++; - m_header_changed = true; + // m_head.info.packages++; + // m_header_changed = true; } // void TppTelnetRec::record_win_size(int width, int height) @@ -93,20 +94,20 @@ void TppTelnetRec::record(ex_u8 type, const ex_u8* data, size_t size) // m_header_changed = true; // } -void TppTelnetRec::record_win_size_startup(int width, int height) -{ - m_head.basic.width = (ex_u16)width; - m_head.basic.height = (ex_u16)height; - m_save_full_header = true; -} - -void TppTelnetRec::record_win_size_change(int width, int height) -{ - TS_RECORD_WIN_SIZE pkg = { 0 }; - pkg.width = (ex_u16)width; - pkg.height = (ex_u16)height; - record(TS_RECORD_TYPE_TELNET_TERM_SIZE, (ex_u8*)&pkg, sizeof(TS_RECORD_WIN_SIZE)); -} +void TppTelnetRec::record_win_size_startup(int width, int height) +{ + m_head.basic.width = (ex_u16)width; + m_head.basic.height = (ex_u16)height; + m_save_full_header = true; +} + +void TppTelnetRec::record_win_size_change(int width, int height) +{ + TS_RECORD_WIN_SIZE pkg = { 0 }; + pkg.width = (ex_u16)width; + pkg.height = (ex_u16)height; + record(TS_RECORD_TYPE_TELNET_TERM_SIZE, (ex_u8*)&pkg, sizeof(TS_RECORD_WIN_SIZE)); +} bool TppTelnetRec::_save_to_info_file() { if (!m_header_changed) diff --git a/server/tp_core/testssh/CMakeLists.txt b/server/tp_core/testssh/CMakeLists.txt index 4a23a43..c5fd10f 100644 --- a/server/tp_core/testssh/CMakeLists.txt +++ b/server/tp_core/testssh/CMakeLists.txt @@ -17,14 +17,17 @@ include_directories( include_directories( ${TP_EXTERNAL_RELEASE_DIR}/include ) -link_directories(${TP_EXTERNAL_RELEASE_DIR}/lib) +link_directories( + ${TP_EXTERNAL_RELEASE_DIR}/lib + ${TP_EXTERNAL_RELEASE_DIR}/lib64 + ) add_executable(testssh ${DIR_SRCS}) if (OS_LINUX) set(CMAKE_EXE_LINKER_FLAGS "-export-dynamic") - target_link_libraries(testssh ssh ssl crypto dl pthread rt util) + target_link_libraries(testssh ssh ssl z crypto dl pthread rt util) elseif (OS_MACOS) - target_link_libraries(testssh ssh ssl crypto dl pthread util) + target_link_libraries(testssh ssh ssl z crypto dl pthread util) endif () diff --git a/server/tp_core/testssh/testssh.cpp b/server/tp_core/testssh/testssh.cpp index 2f934a6..f97809e 100644 --- a/server/tp_core/testssh/testssh.cpp +++ b/server/tp_core/testssh/testssh.cpp @@ -1,7 +1,9 @@ 锘// testssh.cpp : Defines the entry point for the console application. // -#include "stdafx.h" +#ifdef _WIN32 +# include "stdafx.h" +#endif #include #include diff --git a/server/tp_web/src/CMakeLists.txt b/server/tp_web/src/CMakeLists.txt index 33f81ea..c8ba2fb 100644 --- a/server/tp_web/src/CMakeLists.txt +++ b/server/tp_web/src/CMakeLists.txt @@ -1,15 +1,13 @@ cmake_minimum_required(VERSION 3.5) MESSAGE(STATUS "=======================================================") -MESSAGE(STATUS " libtptelnet") +MESSAGE(STATUS " tp_web") MESSAGE(STATUS "=======================================================") #MESSAGE(STATUS "operation system is ${CMAKE_SYSTEM}") #MESSAGE(STATUS "current source directory is ${CMAKE_CURRENT_SOURCE_DIR}") include(../../../CMakeCfg.txt) -#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") - set(CMAKE_EXE_LINKER_FLAGS "-export-dynamic") aux_source_directory(. DIR_SRCS) @@ -23,7 +21,7 @@ include_directories( include_directories( ${TP_EXTERNAL_RELEASE_DIR}/include - ${TP_EXTERNAL_RELEASE_DIR}/include/python + ${TP_EXTERNAL_RELEASE_DIR}/include/python3.7m ) link_directories(${TP_EXTERNAL_RELEASE_DIR}/lib) diff --git a/server/www/packages/packages-linux/x64/PIL/.libs/libfreetype-5766911d.so.6.17.1 b/server/www/packages/packages-linux/x64/PIL/.libs/libfreetype-5766911d.so.6.17.1 new file mode 100755 index 0000000..ac7bae1 Binary files /dev/null and b/server/www/packages/packages-linux/x64/PIL/.libs/libfreetype-5766911d.so.6.17.1 differ diff --git a/server/www/packages/packages-linux/x64/PIL/.libs/libfreetype-7ce95de6.so.6.16.1 b/server/www/packages/packages-linux/x64/PIL/.libs/libfreetype-7ce95de6.so.6.16.1 deleted file mode 100755 index 179f60c..0000000 Binary files a/server/www/packages/packages-linux/x64/PIL/.libs/libfreetype-7ce95de6.so.6.16.1 and /dev/null differ diff --git a/server/www/packages/packages-linux/x64/PIL/.libs/libjpeg-3b10b538.so.9.3.0 b/server/www/packages/packages-linux/x64/PIL/.libs/libjpeg-3b10b538.so.9.3.0 new file mode 100755 index 0000000..19484f6 Binary files /dev/null and b/server/www/packages/packages-linux/x64/PIL/.libs/libjpeg-3b10b538.so.9.3.0 differ diff --git a/server/www/packages/packages-linux/x64/PIL/.libs/libjpeg-3fe7dfc0.so.9.3.0 b/server/www/packages/packages-linux/x64/PIL/.libs/libjpeg-3fe7dfc0.so.9.3.0 deleted file mode 100755 index 835b57b..0000000 Binary files a/server/www/packages/packages-linux/x64/PIL/.libs/libjpeg-3fe7dfc0.so.9.3.0 and /dev/null differ diff --git a/server/www/packages/packages-linux/x64/PIL/.libs/libopenjp2-b3d7668a.so.2.3.1 b/server/www/packages/packages-linux/x64/PIL/.libs/libopenjp2-b3d7668a.so.2.3.1 new file mode 100755 index 0000000..c676915 Binary files /dev/null and b/server/www/packages/packages-linux/x64/PIL/.libs/libopenjp2-b3d7668a.so.2.3.1 differ diff --git a/server/www/packages/packages-linux/x64/PIL/.libs/libopenjp2-e366d6b0.so.2.1.0 b/server/www/packages/packages-linux/x64/PIL/.libs/libopenjp2-e366d6b0.so.2.1.0 deleted file mode 100755 index c31cea4..0000000 Binary files a/server/www/packages/packages-linux/x64/PIL/.libs/libopenjp2-e366d6b0.so.2.1.0 and /dev/null differ diff --git a/server/www/packages/packages-linux/x64/PIL/.libs/libpng16-898afbbd.so.16.35.0 b/server/www/packages/packages-linux/x64/PIL/.libs/libpng16-898afbbd.so.16.35.0 deleted file mode 100755 index 9cf16ab..0000000 Binary files a/server/www/packages/packages-linux/x64/PIL/.libs/libpng16-898afbbd.so.16.35.0 and /dev/null differ diff --git a/server/www/packages/packages-linux/x64/PIL/.libs/libpng16-bedcb7ea.so.16.37.0 b/server/www/packages/packages-linux/x64/PIL/.libs/libpng16-bedcb7ea.so.16.37.0 new file mode 100755 index 0000000..99a0058 Binary files /dev/null and b/server/www/packages/packages-linux/x64/PIL/.libs/libpng16-bedcb7ea.so.16.37.0 differ diff --git a/server/www/packages/packages-linux/x64/PIL/.libs/libtiff-8a6d997d.so.5.3.0 b/server/www/packages/packages-linux/x64/PIL/.libs/libtiff-8267adfe.so.5.4.0 similarity index 52% rename from server/www/packages/packages-linux/x64/PIL/.libs/libtiff-8a6d997d.so.5.3.0 rename to server/www/packages/packages-linux/x64/PIL/.libs/libtiff-8267adfe.so.5.4.0 index ff0a9dd..b0e5d57 100755 Binary files a/server/www/packages/packages-linux/x64/PIL/.libs/libtiff-8a6d997d.so.5.3.0 and b/server/www/packages/packages-linux/x64/PIL/.libs/libtiff-8267adfe.so.5.4.0 differ diff --git a/server/www/packages/packages-linux/x64/PIL/.libs/libwebp-3a2aeecf.so.7.0.5 b/server/www/packages/packages-linux/x64/PIL/.libs/libwebp-3a2aeecf.so.7.0.5 new file mode 100755 index 0000000..61005d4 Binary files /dev/null and b/server/www/packages/packages-linux/x64/PIL/.libs/libwebp-3a2aeecf.so.7.0.5 differ diff --git a/server/www/packages/packages-linux/x64/PIL/.libs/libwebp-8ccd29fd.so.7.0.2 b/server/www/packages/packages-linux/x64/PIL/.libs/libwebp-8ccd29fd.so.7.0.2 deleted file mode 100755 index b2f0763..0000000 Binary files a/server/www/packages/packages-linux/x64/PIL/.libs/libwebp-8ccd29fd.so.7.0.2 and /dev/null differ diff --git a/server/www/packages/packages-linux/x64/PIL/.libs/libwebpdemux-e9ec482e.so.2.0.6 b/server/www/packages/packages-linux/x64/PIL/.libs/libwebpdemux-e9ec482e.so.2.0.6 new file mode 100755 index 0000000..3cde1b8 Binary files /dev/null and b/server/www/packages/packages-linux/x64/PIL/.libs/libwebpdemux-e9ec482e.so.2.0.6 differ diff --git a/server/www/packages/packages-linux/x64/PIL/.libs/libwebpdemux-eba3dc32.so.2.0.4 b/server/www/packages/packages-linux/x64/PIL/.libs/libwebpdemux-eba3dc32.so.2.0.4 deleted file mode 100755 index 66adccb..0000000 Binary files a/server/www/packages/packages-linux/x64/PIL/.libs/libwebpdemux-eba3dc32.so.2.0.4 and /dev/null differ diff --git a/server/www/packages/packages-linux/x64/PIL/.libs/libwebpmux-1c63fe99.so.3.0.2 b/server/www/packages/packages-linux/x64/PIL/.libs/libwebpmux-1c63fe99.so.3.0.2 deleted file mode 100755 index 2045a77..0000000 Binary files a/server/www/packages/packages-linux/x64/PIL/.libs/libwebpmux-1c63fe99.so.3.0.2 and /dev/null differ diff --git a/server/www/packages/packages-linux/x64/PIL/.libs/libwebpmux-40630b44.so.3.0.4 b/server/www/packages/packages-linux/x64/PIL/.libs/libwebpmux-40630b44.so.3.0.4 new file mode 100755 index 0000000..a04c69c Binary files /dev/null and b/server/www/packages/packages-linux/x64/PIL/.libs/libwebpmux-40630b44.so.3.0.4 differ diff --git a/server/www/packages/packages-linux/x64/PIL/.libs/libz-a147dcb0.so.1.2.3 b/server/www/packages/packages-linux/x64/PIL/.libs/libz-a147dcb0.so.1.2.3 index c123f89..1ce02c2 100755 Binary files a/server/www/packages/packages-linux/x64/PIL/.libs/libz-a147dcb0.so.1.2.3 and b/server/www/packages/packages-linux/x64/PIL/.libs/libz-a147dcb0.so.1.2.3 differ diff --git a/server/www/packages/packages-linux/x64/PIL/BdfFontFile.py b/server/www/packages/packages-linux/x64/PIL/BdfFontFile.py index eac19bd..fdf2c09 100644 --- a/server/www/packages/packages-linux/x64/PIL/BdfFontFile.py +++ b/server/www/packages/packages-linux/x64/PIL/BdfFontFile.py @@ -19,8 +19,7 @@ from __future__ import print_function -from . import Image, FontFile - +from . import FontFile, Image # -------------------------------------------------------------------- # parse X Bitmap Distribution Format (BDF) @@ -32,14 +31,10 @@ bdf_slant = { "O": "Oblique", "RI": "Reverse Italic", "RO": "Reverse Oblique", - "OT": "Other" + "OT": "Other", } -bdf_spacing = { - "P": "Proportional", - "M": "Monospaced", - "C": "Cell" -} +bdf_spacing = {"P": "Proportional", "M": "Monospaced", "C": "Cell"} def bdf_char(f): @@ -50,7 +45,7 @@ def bdf_char(f): return None if s[:9] == b"STARTCHAR": break - id = s[9:].strip().decode('ascii') + id = s[9:].strip().decode("ascii") # load symbol properties props = {} @@ -59,7 +54,7 @@ def bdf_char(f): if not s or s[:6] == b"BITMAP": break i = s.find(b" ") - props[s[:i].decode('ascii')] = s[i+1:-1].decode('ascii') + props[s[:i].decode("ascii")] = s[i + 1 : -1].decode("ascii") # load bitmap bitmap = [] @@ -73,7 +68,7 @@ def bdf_char(f): [x, y, l, d] = [int(p) for p in props["BBX"].split()] [dx, dy] = [int(p) for p in props["DWIDTH"].split()] - bbox = (dx, dy), (l, -d-y, x+l, -d), (0, 0, x, y) + bbox = (dx, dy), (l, -d - y, x + l, -d), (0, 0, x, y) try: im = Image.frombytes("1", (x, y), bitmap, "hex", "1") @@ -87,8 +82,8 @@ def bdf_char(f): ## # Font file plugin for the X11 BDF format. -class BdfFontFile(FontFile.FontFile): +class BdfFontFile(FontFile.FontFile): def __init__(self, fp): FontFile.FontFile.__init__(self) @@ -105,10 +100,10 @@ class BdfFontFile(FontFile.FontFile): if not s or s[:13] == b"ENDPROPERTIES": break i = s.find(b" ") - props[s[:i].decode('ascii')] = s[i+1:-1].decode('ascii') + props[s[:i].decode("ascii")] = s[i + 1 : -1].decode("ascii") if s[:i] in [b"COMMENT", b"COPYRIGHT"]: if s.find(b"LogicalFontDescription") < 0: - comments.append(s[i+1:-1].decode('ascii')) + comments.append(s[i + 1 : -1].decode("ascii")) while True: c = bdf_char(fp) diff --git a/server/www/packages/packages-linux/x64/PIL/BlpImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/BlpImagePlugin.py index 398e0fa..7b97964 100644 --- a/server/www/packages/packages-linux/x64/PIL/BlpImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/BlpImagePlugin.py @@ -34,7 +34,6 @@ from io import BytesIO from . import Image, ImageFile - BLP_FORMAT_JPEG = 0 BLP_ENCODING_UNCOMPRESSED = 1 @@ -47,11 +46,7 @@ BLP_ALPHA_ENCODING_DXT5 = 7 def unpack_565(i): - return ( - ((i >> 11) & 0x1f) << 3, - ((i >> 5) & 0x3f) << 2, - (i & 0x1f) << 3 - ) + return (((i >> 11) & 0x1F) << 3, ((i >> 5) & 0x3F) << 2, (i & 0x1F) << 3) def decode_dxt1(data, alpha=False): @@ -119,7 +114,7 @@ def decode_dxt3(data): for block in range(blocks): idx = block * 16 - block = data[idx:idx + 16] + block = data[idx : idx + 16] # Decode next 16-byte block. bits = struct.unpack_from("<8B", block) color0, color1 = struct.unpack_from(">= 4 else: high = True - a &= 0xf + a &= 0xF a *= 17 # We get a value between 0 and 15 color_code = (code >> 2 * (4 * j + i)) & 0x03 @@ -172,14 +167,12 @@ def decode_dxt5(data): for block in range(blocks): idx = block * 16 - block = data[idx:idx + 16] + block = data[idx : idx + 16] # Decode next 16-byte block. a0, a1 = struct.unpack_from("= 40: # v3 and OS/2 - file_info['y_flip'] = i8(header_data[7]) == 0xff - file_info['direction'] = 1 if file_info['y_flip'] else -1 - file_info['width'] = i32(header_data[0:4]) - file_info['height'] = (i32(header_data[4:8]) - if not file_info['y_flip'] - else 2**32 - i32(header_data[4:8])) - file_info['planes'] = i16(header_data[8:10]) - file_info['bits'] = i16(header_data[10:12]) - file_info['compression'] = i32(header_data[12:16]) - # byte size of pixel data - file_info['data_size'] = i32(header_data[16:20]) - file_info['pixels_per_meter'] = (i32(header_data[20:24]), - i32(header_data[24:28])) - file_info['colors'] = i32(header_data[28:32]) - file_info['palette_padding'] = 4 - self.info["dpi"] = tuple( - map(lambda x: int(math.ceil(x / 39.3701)), - file_info['pixels_per_meter'])) - if file_info['compression'] == self.BITFIELDS: - if len(header_data) >= 52: - for idx, mask in enumerate(['r_mask', - 'g_mask', - 'b_mask', - 'a_mask']): - file_info[mask] = i32(header_data[36+idx*4:40+idx*4]) - else: - # 40 byte headers only have the three components in the - # bitfields masks, - # ref: https://msdn.microsoft.com/en-us/library/windows/desktop/dd183376(v=vs.85).aspx - # See also https://github.com/python-pillow/Pillow/issues/1293 - # There is a 4th component in the RGBQuad, in the alpha - # location, but it is listed as a reserved component, - # and it is not generally an alpha channel - file_info['a_mask'] = 0x0 - for mask in ['r_mask', 'g_mask', 'b_mask']: - file_info[mask] = i32(read(4)) - file_info['rgb_mask'] = (file_info['r_mask'], - file_info['g_mask'], - file_info['b_mask']) - file_info['rgba_mask'] = (file_info['r_mask'], - file_info['g_mask'], - file_info['b_mask'], - file_info['a_mask']) + header_data = ImageFile._safe_read(self.fp, file_info["header_size"] - 4) + + # -------------------------------------------------- IBM OS/2 Bitmap v1 + # ----- This format has different offsets because of width/height types + if file_info["header_size"] == 12: + file_info["width"] = i16(header_data[0:2]) + file_info["height"] = i16(header_data[2:4]) + file_info["planes"] = i16(header_data[4:6]) + file_info["bits"] = i16(header_data[6:8]) + file_info["compression"] = self.RAW + file_info["palette_padding"] = 3 + + # --------------------------------------------- Windows Bitmap v2 to v5 + # v3, OS/2 v2, v4, v5 + elif file_info["header_size"] in (40, 64, 108, 124): + file_info["y_flip"] = i8(header_data[7]) == 0xFF + file_info["direction"] = 1 if file_info["y_flip"] else -1 + file_info["width"] = i32(header_data[0:4]) + file_info["height"] = ( + i32(header_data[4:8]) + if not file_info["y_flip"] + else 2 ** 32 - i32(header_data[4:8]) + ) + file_info["planes"] = i16(header_data[8:10]) + file_info["bits"] = i16(header_data[10:12]) + file_info["compression"] = i32(header_data[12:16]) + # byte size of pixel data + file_info["data_size"] = i32(header_data[16:20]) + file_info["pixels_per_meter"] = ( + i32(header_data[20:24]), + i32(header_data[24:28]), + ) + file_info["colors"] = i32(header_data[28:32]) + file_info["palette_padding"] = 4 + self.info["dpi"] = tuple( + int(x / 39.3701 + 0.5) for x in file_info["pixels_per_meter"] + ) + if file_info["compression"] == self.BITFIELDS: + if len(header_data) >= 52: + for idx, mask in enumerate( + ["r_mask", "g_mask", "b_mask", "a_mask"] + ): + file_info[mask] = i32(header_data[36 + idx * 4 : 40 + idx * 4]) + else: + # 40 byte headers only have the three components in the + # bitfields masks, ref: + # https://msdn.microsoft.com/en-us/library/windows/desktop/dd183376(v=vs.85).aspx + # See also + # https://github.com/python-pillow/Pillow/issues/1293 + # There is a 4th component in the RGBQuad, in the alpha + # location, but it is listed as a reserved component, + # and it is not generally an alpha channel + file_info["a_mask"] = 0x0 + for mask in ["r_mask", "g_mask", "b_mask"]: + file_info[mask] = i32(read(4)) + file_info["rgb_mask"] = ( + file_info["r_mask"], + file_info["g_mask"], + file_info["b_mask"], + ) + file_info["rgba_mask"] = ( + file_info["r_mask"], + file_info["g_mask"], + file_info["b_mask"], + file_info["a_mask"], + ) else: - raise IOError("Unsupported BMP header type (%d)" % - file_info['header_size']) + raise IOError("Unsupported BMP header type (%d)" % file_info["header_size"]) + # ------------------ Special case : header is reported 40, which # ---------------------- is shorter than real size for bpp >= 16 - self._size = file_info['width'], file_info['height'] - # -------- If color count was not found in the header, compute from bits - file_info['colors'] = file_info['colors'] if file_info.get('colors', 0) else (1 << file_info['bits']) - # -------------------------------- Check abnormal values for DOS attacks - if file_info['width'] * file_info['height'] > 2**31: + self._size = file_info["width"], file_info["height"] + + # ------- If color count was not found in the header, compute from bits + file_info["colors"] = ( + file_info["colors"] + if file_info.get("colors", 0) + else (1 << file_info["bits"]) + ) + + # ------------------------------- Check abnormal values for DOS attacks + if file_info["width"] * file_info["height"] > 2 ** 31: raise IOError("Unsupported BMP Size: (%dx%d)" % self.size) - # ----------------------- Check bit depth for unusual unsupported values - self.mode, raw_mode = BIT2MODE.get(file_info['bits'], (None, None)) + + # ---------------------- Check bit depth for unusual unsupported values + self.mode, raw_mode = BIT2MODE.get(file_info["bits"], (None, None)) if self.mode is None: - raise IOError("Unsupported BMP pixel depth (%d)" - % file_info['bits']) - # ----------------- Process BMP with Bitfields compression (not palette) - if file_info['compression'] == self.BITFIELDS: + raise IOError("Unsupported BMP pixel depth (%d)" % file_info["bits"]) + + # ---------------- Process BMP with Bitfields compression (not palette) + if file_info["compression"] == self.BITFIELDS: SUPPORTED = { - 32: [(0xff0000, 0xff00, 0xff, 0x0), - (0xff0000, 0xff00, 0xff, 0xff000000), - (0x0, 0x0, 0x0, 0x0), - (0xff000000, 0xff0000, 0xff00, 0x0)], - 24: [(0xff0000, 0xff00, 0xff)], - 16: [(0xf800, 0x7e0, 0x1f), (0x7c00, 0x3e0, 0x1f)] + 32: [ + (0xFF0000, 0xFF00, 0xFF, 0x0), + (0xFF0000, 0xFF00, 0xFF, 0xFF000000), + (0xFF, 0xFF00, 0xFF0000, 0xFF000000), + (0x0, 0x0, 0x0, 0x0), + (0xFF000000, 0xFF0000, 0xFF00, 0x0), + ], + 24: [(0xFF0000, 0xFF00, 0xFF)], + 16: [(0xF800, 0x7E0, 0x1F), (0x7C00, 0x3E0, 0x1F)], } MASK_MODES = { - (32, (0xff0000, 0xff00, 0xff, 0x0)): "BGRX", - (32, (0xff000000, 0xff0000, 0xff00, 0x0)): "XBGR", - (32, (0xff0000, 0xff00, 0xff, 0xff000000)): "BGRA", + (32, (0xFF0000, 0xFF00, 0xFF, 0x0)): "BGRX", + (32, (0xFF000000, 0xFF0000, 0xFF00, 0x0)): "XBGR", + (32, (0xFF, 0xFF00, 0xFF0000, 0xFF000000)): "RGBA", + (32, (0xFF0000, 0xFF00, 0xFF, 0xFF000000)): "BGRA", (32, (0x0, 0x0, 0x0, 0x0)): "BGRA", - (24, (0xff0000, 0xff00, 0xff)): "BGR", - (16, (0xf800, 0x7e0, 0x1f)): "BGR;16", - (16, (0x7c00, 0x3e0, 0x1f)): "BGR;15" + (24, (0xFF0000, 0xFF00, 0xFF)): "BGR", + (16, (0xF800, 0x7E0, 0x1F)): "BGR;16", + (16, (0x7C00, 0x3E0, 0x1F)): "BGR;15", } - if file_info['bits'] in SUPPORTED: - if file_info['bits'] == 32 and \ - file_info['rgba_mask'] in SUPPORTED[file_info['bits']]: - raw_mode = MASK_MODES[(file_info['bits'], file_info['rgba_mask'])] - self.mode = "RGBA" if raw_mode in ("BGRA",) else self.mode - elif (file_info['bits'] in (24, 16) and - file_info['rgb_mask'] in SUPPORTED[file_info['bits']]): - raw_mode = MASK_MODES[ - (file_info['bits'], file_info['rgb_mask']) - ] + if file_info["bits"] in SUPPORTED: + if ( + file_info["bits"] == 32 + and file_info["rgba_mask"] in SUPPORTED[file_info["bits"]] + ): + raw_mode = MASK_MODES[(file_info["bits"], file_info["rgba_mask"])] + self.mode = "RGBA" if "A" in raw_mode else self.mode + elif ( + file_info["bits"] in (24, 16) + and file_info["rgb_mask"] in SUPPORTED[file_info["bits"]] + ): + raw_mode = MASK_MODES[(file_info["bits"], file_info["rgb_mask"])] else: raise IOError("Unsupported BMP bitfields layout") else: raise IOError("Unsupported BMP bitfields layout") - elif file_info['compression'] == self.RAW: - if file_info['bits'] == 32 and header == 22: # 32-bit .cur offset + elif file_info["compression"] == self.RAW: + if file_info["bits"] == 32 and header == 22: # 32-bit .cur offset raw_mode, self.mode = "BGRA", "RGBA" else: - raise IOError("Unsupported BMP compression (%d)" % - file_info['compression']) - # ---------------- Once the header is processed, process the palette/LUT + raise IOError("Unsupported BMP compression (%d)" % file_info["compression"]) + + # --------------- Once the header is processed, process the palette/LUT if self.mode == "P": # Paletted for 1, 4 and 8 bit images - # ----------------------------------------------------- 1-bit images - if not (0 < file_info['colors'] <= 65536): - raise IOError("Unsupported BMP Palette size (%d)" % - file_info['colors']) + + # ---------------------------------------------------- 1-bit images + if not (0 < file_info["colors"] <= 65536): + raise IOError("Unsupported BMP Palette size (%d)" % file_info["colors"]) else: - padding = file_info['palette_padding'] - palette = read(padding * file_info['colors']) + padding = file_info["palette_padding"] + palette = read(padding * file_info["colors"]) greyscale = True - indices = (0, 255) if file_info['colors'] == 2 else \ - list(range(file_info['colors'])) - # ------------------ Check if greyscale and ignore palette if so + indices = ( + (0, 255) + if file_info["colors"] == 2 + else list(range(file_info["colors"])) + ) + + # ----------------- Check if greyscale and ignore palette if so for ind, val in enumerate(indices): - rgb = palette[ind*padding:ind*padding + 3] + rgb = palette[ind * padding : ind * padding + 3] if rgb != o8(val) * 3: greyscale = False - # -------- If all colors are grey, white or black, ditch palette + + # ------- If all colors are grey, white or black, ditch palette if greyscale: - self.mode = "1" if file_info['colors'] == 2 else "L" + self.mode = "1" if file_info["colors"] == 2 else "L" raw_mode = self.mode else: self.mode = "P" self.palette = ImagePalette.raw( - "BGRX" if padding == 4 else "BGR", palette) + "BGRX" if padding == 4 else "BGR", palette + ) - # ----------------------------- Finally set the tile data for the plugin - self.info['compression'] = file_info['compression'] + # ---------------------------- Finally set the tile data for the plugin + self.info["compression"] = file_info["compression"] self.tile = [ - ('raw', - (0, 0, file_info['width'], file_info['height']), - offset or self.fp.tell(), - (raw_mode, - ((file_info['width'] * file_info['bits'] + 31) >> 3) & (~3), - file_info['direction'])) + ( + "raw", + (0, 0, file_info["width"], file_info["height"]), + offset or self.fp.tell(), + ( + raw_mode, + ((file_info["width"] * file_info["bits"] + 31) >> 3) & (~3), + file_info["direction"], + ), + ) ] def _open(self): @@ -243,9 +275,9 @@ class BmpImageFile(ImageFile.ImageFile): self._bitmap(offset=offset) -# ============================================================================== +# ============================================================================= # Image plugin for the DIB format (BMP alias) -# ============================================================================== +# ============================================================================= class DibImageFile(BmpImageFile): format = "DIB" @@ -254,6 +286,7 @@ class DibImageFile(BmpImageFile): def _open(self): self._bitmap() + # # -------------------------------------------------------------------- # Write BMP file @@ -268,7 +301,11 @@ SAVE = { } -def _save(im, fp, filename): +def _dib_save(im, fp, filename): + _save(im, fp, filename, False) + + +def _save(im, fp, filename, bitmap_header=True): try: rawmode, bits, colors = SAVE[im.mode] except KeyError: @@ -279,32 +316,38 @@ def _save(im, fp, filename): dpi = info.get("dpi", (96, 96)) # 1 meter == 39.3701 inches - ppm = tuple(map(lambda x: int(x * 39.3701), dpi)) + ppm = tuple(map(lambda x: int(x * 39.3701 + 0.5), dpi)) - stride = ((im.size[0]*bits+7)//8+3) & (~3) + stride = ((im.size[0] * bits + 7) // 8 + 3) & (~3) header = 40 # or 64 for OS/2 version 2 - offset = 14 + header + colors * 4 image = stride * im.size[1] # bitmap header - fp.write(b"BM" + # file type (magic) - o32(offset+image) + # file size - o32(0) + # reserved - o32(offset)) # image data offset + if bitmap_header: + offset = 14 + header + colors * 4 + fp.write( + b"BM" + + o32(offset + image) # file type (magic) + + o32(0) # file size + + o32(offset) # reserved + ) # image data offset # bitmap info header - fp.write(o32(header) + # info header size - o32(im.size[0]) + # width - o32(im.size[1]) + # height - o16(1) + # planes - o16(bits) + # depth - o32(0) + # compression (0=uncompressed) - o32(image) + # size of bitmap - o32(ppm[0]) + o32(ppm[1]) + # resolution - o32(colors) + # colors used - o32(colors)) # colors important + fp.write( + o32(header) # info header size + + o32(im.size[0]) # width + + o32(im.size[1]) # height + + o16(1) # planes + + o16(bits) # depth + + o32(0) # compression (0=uncompressed) + + o32(image) # size of bitmap + + o32(ppm[0]) # resolution + + o32(ppm[1]) # resolution + + o32(colors) # colors used + + o32(colors) # colors important + ) - fp.write(b"\0" * (header - 40)) # padding (for OS/2 format) + fp.write(b"\0" * (header - 40)) # padding (for OS/2 format) if im.mode == "1": for i in (0, 255): @@ -315,8 +358,8 @@ def _save(im, fp, filename): elif im.mode == "P": fp.write(im.im.getpalette("RGB", "BGRX")) - ImageFile._save(im, fp, [("raw", (0, 0)+im.size, 0, - (rawmode, stride, -1))]) + ImageFile._save(im, fp, [("raw", (0, 0) + im.size, 0, (rawmode, stride, -1))]) + # # -------------------------------------------------------------------- @@ -329,3 +372,10 @@ Image.register_save(BmpImageFile.format, _save) Image.register_extension(BmpImageFile.format, ".bmp") Image.register_mime(BmpImageFile.format, "image/bmp") + +Image.register_open(DibImageFile.format, DibImageFile, _dib_accept) +Image.register_save(DibImageFile.format, _dib_save) + +Image.register_extension(DibImageFile.format, ".dib") + +Image.register_mime(DibImageFile.format, "image/bmp") diff --git a/server/www/packages/packages-linux/x64/PIL/BufrStubImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/BufrStubImagePlugin.py index a1957b3..56cac3b 100644 --- a/server/www/packages/packages-linux/x64/PIL/BufrStubImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/BufrStubImagePlugin.py @@ -27,6 +27,7 @@ def register_handler(handler): # -------------------------------------------------------------------- # Image adapter + def _accept(prefix): return prefix[:4] == b"BUFR" or prefix[:4] == b"ZCZC" diff --git a/server/www/packages/packages-linux/x64/PIL/ContainerIO.py b/server/www/packages/packages-linux/x64/PIL/ContainerIO.py index 682ad90..3cf9d82 100644 --- a/server/www/packages/packages-linux/x64/PIL/ContainerIO.py +++ b/server/www/packages/packages-linux/x64/PIL/ContainerIO.py @@ -18,9 +18,10 @@ # A file object that provides read access to a part of an existing # file (for example a TAR file). +import io + class ContainerIO(object): - def __init__(self, file, offset, length): """ Create file object. @@ -39,9 +40,9 @@ class ContainerIO(object): # Always false. def isatty(self): - return 0 + return False - def seek(self, offset, mode=0): + def seek(self, offset, mode=io.SEEK_SET): """ Move file pointer. diff --git a/server/www/packages/packages-linux/x64/PIL/CurImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/CurImagePlugin.py index 3e8f321..9e2d8c9 100644 --- a/server/www/packages/packages-linux/x64/PIL/CurImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/CurImagePlugin.py @@ -18,9 +18,11 @@ from __future__ import print_function -from . import Image, BmpImagePlugin +from . import BmpImagePlugin, Image from ._binary import i8, i16le as i16, i32le as i32 +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.1" # @@ -34,6 +36,7 @@ def _accept(prefix): ## # Image plugin for Windows Cursor files. + class CurImageFile(BmpImagePlugin.BmpImageFile): format = "CUR" @@ -63,9 +66,9 @@ class CurImageFile(BmpImagePlugin.BmpImageFile): self._bitmap(i32(m[12:]) + offset) # patch up the bitmap height - self._size = self.size[0], self.size[1]//2 + self._size = self.size[0], self.size[1] // 2 d, e, o, a = self.tile[0] - self.tile[0] = d, (0, 0)+self.size, o, a + self.tile[0] = d, (0, 0) + self.size, o, a return diff --git a/server/www/packages/packages-linux/x64/PIL/DcxImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/DcxImagePlugin.py index 2045927..57c3214 100644 --- a/server/www/packages/packages-linux/x64/PIL/DcxImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/DcxImagePlugin.py @@ -25,6 +25,8 @@ from . import Image from ._binary import i32le as i32 from .PcxImagePlugin import PcxImageFile +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.2" MAGIC = 0x3ADE68B1 # QUIZ: what's this value, then? @@ -37,6 +39,7 @@ def _accept(prefix): ## # Image plugin for the Intel DCX format. + class DcxImageFile(PcxImageFile): format = "DCX" @@ -81,6 +84,15 @@ class DcxImageFile(PcxImageFile): def tell(self): return self.frame + def _close__fp(self): + try: + if self.__fp != self.fp: + self.__fp.close() + except AttributeError: + pass + finally: + self.__fp = None + Image.register_open(DcxImageFile.format, DcxImageFile, _accept) diff --git a/server/www/packages/packages-linux/x64/PIL/DdsImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/DdsImagePlugin.py index 7660827..b2d5089 100644 --- a/server/www/packages/packages-linux/x64/PIL/DdsImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/DdsImagePlugin.py @@ -12,8 +12,8 @@ Full text of the CC0 license: import struct from io import BytesIO -from . import Image, ImageFile +from . import Image, ImageFile # Magic ("DDS ") DDS_MAGIC = 0x20534444 @@ -61,8 +61,7 @@ DDS_LUMINANCEA = DDPF_LUMINANCE | DDPF_ALPHAPIXELS DDS_ALPHA = DDPF_ALPHA DDS_PAL8 = DDPF_PALETTEINDEXED8 -DDS_HEADER_FLAGS_TEXTURE = (DDSD_CAPS | DDSD_HEIGHT | DDSD_WIDTH | - DDSD_PIXELFORMAT) +DDS_HEADER_FLAGS_TEXTURE = DDSD_CAPS | DDSD_HEIGHT | DDSD_WIDTH | DDSD_PIXELFORMAT DDS_HEADER_FLAGS_MIPMAP = DDSD_MIPMAPCOUNT DDS_HEADER_FLAGS_VOLUME = DDSD_DEPTH DDS_HEADER_FLAGS_PITCH = DDSD_PITCH @@ -118,48 +117,54 @@ class DdsImageFile(ImageFile.ImageFile): self.mode = "RGBA" pitch, depth, mipmaps = struct.unpack("<3I", header.read(12)) - reserved = struct.unpack("<11I", header.read(44)) + struct.unpack("<11I", header.read(44)) # reserved # pixel format pfsize, pfflags = struct.unpack("<2I", header.read(8)) fourcc = header.read(4) - bitcount, rmask, gmask, bmask, amask = struct.unpack("<5I", - header.read(20)) + bitcount, = struct.unpack(" 0: - s = fp.read(min(lengthfile, 100*1024)) + s = fp.read(min(lengthfile, 100 * 1024)) if not s: break lengthfile -= len(s) f.write(s) # Build Ghostscript command - command = ["gs", - "-q", # quiet mode - "-g%dx%d" % size, # set output geometry (pixels) - "-r%fx%f" % res, # set input DPI (dots per inch) - "-dBATCH", # exit after processing - "-dNOPAUSE", # don't pause between pages - "-dSAFER", # safe mode - "-sDEVICE=ppmraw", # ppm driver - "-sOutputFile=%s" % outfile, # output file - "-c", "%d %d translate" % (-bbox[0], -bbox[1]), - # adjust for image origin - "-f", infile, # input file - "-c", "showpage", # showpage (see: https://bugs.ghostscript.com/show_bug.cgi?id=698272) - ] + command = [ + "gs", + "-q", # quiet mode + "-g%dx%d" % size, # set output geometry (pixels) + "-r%fx%f" % res, # set input DPI (dots per inch) + "-dBATCH", # exit after processing + "-dNOPAUSE", # don't pause between pages + "-dSAFER", # safe mode + "-sDEVICE=ppmraw", # ppm driver + "-sOutputFile=%s" % outfile, # output file + # adjust for image origin + "-c", + "%d %d translate" % (-bbox[0], -bbox[1]), + "-f", + infile, # input file + # showpage (see https://bugs.ghostscript.com/show_bug.cgi?id=698272) + "-c", + "showpage", + ] if gs_windows_binary is not None: if not gs_windows_binary: - raise WindowsError('Unable to locate Ghostscript on paths') + raise WindowsError("Unable to locate Ghostscript on paths") command[0] = gs_windows_binary # push data through Ghostscript try: - with open(os.devnull, 'w+b') as devnull: - startupinfo = None - if sys.platform.startswith('win'): - startupinfo = subprocess.STARTUPINFO() - startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW - subprocess.check_call(command, stdin=devnull, stdout=devnull, - startupinfo=startupinfo) + startupinfo = None + if sys.platform.startswith("win"): + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + subprocess.check_call(command, startupinfo=startupinfo) im = Image.open(outfile) im.load() finally: @@ -162,11 +173,12 @@ class PSFile(object): """ Wrapper for bytesio object that treats either CR or LF as end of line. """ + def __init__(self, fp): self.fp = fp self.char = None - def seek(self, offset, whence=0): + def seek(self, offset, whence=io.SEEK_SET): self.char = None self.fp.seek(offset, whence) @@ -184,12 +196,12 @@ class PSFile(object): if self.char in b"\r\n": self.char = None - return s.decode('latin-1') + return s.decode("latin-1") def _accept(prefix): - return prefix[:4] == b"%!PS" or \ - (len(prefix) >= 4 and i32(prefix) == 0xC6D3D0C5) + return prefix[:4] == b"%!PS" or (len(prefix) >= 4 and i32(prefix) == 0xC6D3D0C5) + ## # Image plugin for Encapsulated Postscript. This plugin supports only @@ -223,7 +235,7 @@ class EpsImageFile(ImageFile.ImageFile): # Load EPS header s_raw = fp.readline() - s = s_raw.strip('\r\n') + s = s_raw.strip("\r\n") while s_raw: if s: @@ -245,8 +257,9 @@ class EpsImageFile(ImageFile.ImageFile): # put floating point values there anyway. box = [int(float(i)) for i in v.split()] self._size = box[2] - box[0], box[3] - box[1] - self.tile = [("eps", (0, 0) + self.size, offset, - (length, box))] + self.tile = [ + ("eps", (0, 0) + self.size, offset, (length, box)) + ] except Exception: pass @@ -261,7 +274,7 @@ class EpsImageFile(ImageFile.ImageFile): self.info[k[:8]] = k[9:] else: self.info[k] = "" - elif s[0] == '%': + elif s[0] == "%": # handle non-DSC Postscript comments that some # tools mistakenly put in the Comments section pass @@ -269,7 +282,7 @@ class EpsImageFile(ImageFile.ImageFile): raise IOError("bad EPS header") s_raw = fp.readline() - s = s_raw.strip('\r\n') + s = s_raw.strip("\r\n") if s and s[:1] != "%": break @@ -296,7 +309,7 @@ class EpsImageFile(ImageFile.ImageFile): self._size = int(x), int(y) return - s = fp.readline().strip('\r\n') + s = fp.readline().strip("\r\n") if not s: break @@ -309,7 +322,7 @@ class EpsImageFile(ImageFile.ImageFile): if s[:4] == b"%!PS": # for HEAD without binary preview - fp.seek(0, 2) + fp.seek(0, io.SEEK_END) length = fp.tell() offset = 0 elif i32(s[0:4]) == 0xC6D3D0C5: @@ -343,6 +356,7 @@ class EpsImageFile(ImageFile.ImageFile): # # -------------------------------------------------------------------- + def _save(im, fp, filename, eps=1): """EPS Writer for the Python Imaging Library.""" @@ -365,7 +379,7 @@ def _save(im, fp, filename, eps=1): wrapped_fp = False if fp != sys.stdout: if sys.version_info.major > 2: - fp = io.TextIOWrapper(fp, encoding='latin-1') + fp = io.TextIOWrapper(fp, encoding="latin-1") wrapped_fp = True try: @@ -380,7 +394,7 @@ def _save(im, fp, filename, eps=1): fp.write("%%EndComments\n") fp.write("%%Page: 1 1\n") fp.write("%%ImageData: %d %d " % im.size) - fp.write("%d %d 0 1 1 \"%s\"\n" % operator) + fp.write('%d %d 0 1 1 "%s"\n' % operator) # # image header @@ -395,7 +409,7 @@ def _save(im, fp, filename, eps=1): if hasattr(fp, "flush"): fp.flush() - ImageFile._save(im, base_fp, [("eps", (0, 0)+im.size, 0, None)]) + ImageFile._save(im, base_fp, [("eps", (0, 0) + im.size, 0, None)]) fp.write("\n%%%%EndBinary\n") fp.write("grestore end\n") @@ -405,6 +419,7 @@ def _save(im, fp, filename, eps=1): if wrapped_fp: fp.detach() + # # -------------------------------------------------------------------- diff --git a/server/www/packages/packages-linux/x64/PIL/ExifTags.py b/server/www/packages/packages-linux/x64/PIL/ExifTags.py index a8ad26b..47a981e 100644 --- a/server/www/packages/packages-linux/x64/PIL/ExifTags.py +++ b/server/www/packages/packages-linux/x64/PIL/ExifTags.py @@ -18,11 +18,10 @@ # Maps EXIF tags to tag names. TAGS = { - # possibly incomplete - 0x000b: "ProcessingSoftware", - 0x00fe: "NewSubfileType", - 0x00ff: "SubfileType", + 0x000B: "ProcessingSoftware", + 0x00FE: "NewSubfileType", + 0x00FF: "SubfileType", 0x0100: "ImageWidth", 0x0101: "ImageLength", 0x0102: "BitsPerSample", @@ -31,10 +30,10 @@ TAGS = { 0x0107: "Thresholding", 0x0108: "CellWidth", 0x0109: "CellLength", - 0x010a: "FillOrder", - 0x010d: "DocumentName", - 0x010e: "ImageDescription", - 0x010f: "Make", + 0x010A: "FillOrder", + 0x010D: "DocumentName", + 0x010E: "ImageDescription", + 0x010F: "Make", 0x0110: "Model", 0x0111: "StripOffsets", 0x0112: "Orientation", @@ -43,10 +42,10 @@ TAGS = { 0x0117: "StripByteCounts", 0x0118: "MinSampleValue", 0x0119: "MaxSampleValue", - 0x011a: "XResolution", - 0x011b: "YResolution", - 0x011c: "PlanarConfiguration", - 0x011d: "PageName", + 0x011A: "XResolution", + 0x011B: "YResolution", + 0x011C: "PlanarConfiguration", + 0x011D: "PageName", 0x0120: "FreeOffsets", 0x0121: "FreeByteCounts", 0x0122: "GrayResponseUnit", @@ -55,24 +54,24 @@ TAGS = { 0x0125: "T6Options", 0x0128: "ResolutionUnit", 0x0129: "PageNumber", - 0x012d: "TransferFunction", + 0x012D: "TransferFunction", 0x0131: "Software", 0x0132: "DateTime", - 0x013b: "Artist", - 0x013c: "HostComputer", - 0x013d: "Predictor", - 0x013e: "WhitePoint", - 0x013f: "PrimaryChromaticities", + 0x013B: "Artist", + 0x013C: "HostComputer", + 0x013D: "Predictor", + 0x013E: "WhitePoint", + 0x013F: "PrimaryChromaticities", 0x0140: "ColorMap", 0x0141: "HalftoneHints", 0x0142: "TileWidth", 0x0143: "TileLength", 0x0144: "TileOffsets", 0x0145: "TileByteCounts", - 0x014a: "SubIFDs", - 0x014c: "InkSet", - 0x014d: "InkNames", - 0x014e: "NumberOfInks", + 0x014A: "SubIFDs", + 0x014C: "InkSet", + 0x014D: "InkNames", + 0x014E: "NumberOfInks", 0x0150: "DotRange", 0x0151: "TargetPrinter", 0x0152: "ExtraSamples", @@ -83,9 +82,9 @@ TAGS = { 0x0157: "ClipPath", 0x0158: "XClipPathUnits", 0x0159: "YClipPathUnits", - 0x015a: "Indexed", - 0x015b: "JPEGTables", - 0x015f: "OPIProxy", + 0x015A: "Indexed", + 0x015B: "JPEGTables", + 0x015F: "OPIProxy", 0x0200: "JPEGProc", 0x0201: "JpegIFOffset", 0x0202: "JpegIFByteCount", @@ -99,20 +98,20 @@ TAGS = { 0x0212: "YCbCrSubSampling", 0x0213: "YCbCrPositioning", 0x0214: "ReferenceBlackWhite", - 0x02bc: "XMLPacket", + 0x02BC: "XMLPacket", 0x1000: "RelatedImageFileFormat", 0x1001: "RelatedImageWidth", 0x1002: "RelatedImageLength", 0x4746: "Rating", 0x4749: "RatingPercent", - 0x800d: "ImageID", - 0x828d: "CFARepeatPatternDim", - 0x828e: "CFAPattern", - 0x828f: "BatteryLevel", + 0x800D: "ImageID", + 0x828D: "CFARepeatPatternDim", + 0x828E: "CFAPattern", + 0x828F: "BatteryLevel", 0x8298: "Copyright", - 0x829a: "ExposureTime", - 0x829d: "FNumber", - 0x83bb: "IPTCNAA", + 0x829A: "ExposureTime", + 0x829D: "FNumber", + 0x83BB: "IPTCNAA", 0x8649: "ImageResources", 0x8769: "ExifOffset", 0x8773: "InterColorProfile", @@ -122,8 +121,8 @@ TAGS = { 0x8827: "ISOSpeedRatings", 0x8828: "OECF", 0x8829: "Interlace", - 0x882a: "TimeZoneOffset", - 0x882b: "SelfTimerMode", + 0x882A: "TimeZoneOffset", + 0x882B: "SelfTimerMode", 0x9000: "ExifVersion", 0x9003: "DateTimeOriginal", 0x9004: "DateTimeDigitized", @@ -138,142 +137,142 @@ TAGS = { 0x9207: "MeteringMode", 0x9208: "LightSource", 0x9209: "Flash", - 0x920a: "FocalLength", - 0x920b: "FlashEnergy", - 0x920c: "SpatialFrequencyResponse", - 0x920d: "Noise", + 0x920A: "FocalLength", + 0x920B: "FlashEnergy", + 0x920C: "SpatialFrequencyResponse", + 0x920D: "Noise", 0x9211: "ImageNumber", 0x9212: "SecurityClassification", 0x9213: "ImageHistory", 0x9214: "SubjectLocation", 0x9215: "ExposureIndex", 0x9216: "TIFF/EPStandardID", - 0x927c: "MakerNote", + 0x927C: "MakerNote", 0x9286: "UserComment", 0x9290: "SubsecTime", 0x9291: "SubsecTimeOriginal", 0x9292: "SubsecTimeDigitized", - 0x9c9b: "XPTitle", - 0x9c9c: "XPComment", - 0x9c9d: "XPAuthor", - 0x9c9e: "XPKeywords", - 0x9c9f: "XPSubject", - 0xa000: "FlashPixVersion", - 0xa001: "ColorSpace", - 0xa002: "ExifImageWidth", - 0xa003: "ExifImageHeight", - 0xa004: "RelatedSoundFile", - 0xa005: "ExifInteroperabilityOffset", - 0xa20b: "FlashEnergy", - 0xa20c: "SpatialFrequencyResponse", - 0xa20e: "FocalPlaneXResolution", - 0xa20f: "FocalPlaneYResolution", - 0xa210: "FocalPlaneResolutionUnit", - 0xa214: "SubjectLocation", - 0xa215: "ExposureIndex", - 0xa217: "SensingMethod", - 0xa300: "FileSource", - 0xa301: "SceneType", - 0xa302: "CFAPattern", - 0xa401: "CustomRendered", - 0xa402: "ExposureMode", - 0xa403: "WhiteBalance", - 0xa404: "DigitalZoomRatio", - 0xa405: "FocalLengthIn35mmFilm", - 0xa406: "SceneCaptureType", - 0xa407: "GainControl", - 0xa408: "Contrast", - 0xa409: "Saturation", - 0xa40a: "Sharpness", - 0xa40b: "DeviceSettingDescription", - 0xa40c: "SubjectDistanceRange", - 0xa420: "ImageUniqueID", - 0xa430: "CameraOwnerName", - 0xa431: "BodySerialNumber", - 0xa432: "LensSpecification", - 0xa433: "LensMake", - 0xa434: "LensModel", - 0xa435: "LensSerialNumber", - 0xa500: "Gamma", - 0xc4a5: "PrintImageMatching", - 0xc612: "DNGVersion", - 0xc613: "DNGBackwardVersion", - 0xc614: "UniqueCameraModel", - 0xc615: "LocalizedCameraModel", - 0xc616: "CFAPlaneColor", - 0xc617: "CFALayout", - 0xc618: "LinearizationTable", - 0xc619: "BlackLevelRepeatDim", - 0xc61a: "BlackLevel", - 0xc61b: "BlackLevelDeltaH", - 0xc61c: "BlackLevelDeltaV", - 0xc61d: "WhiteLevel", - 0xc61e: "DefaultScale", - 0xc61f: "DefaultCropOrigin", - 0xc620: "DefaultCropSize", - 0xc621: "ColorMatrix1", - 0xc622: "ColorMatrix2", - 0xc623: "CameraCalibration1", - 0xc624: "CameraCalibration2", - 0xc625: "ReductionMatrix1", - 0xc626: "ReductionMatrix2", - 0xc627: "AnalogBalance", - 0xc628: "AsShotNeutral", - 0xc629: "AsShotWhiteXY", - 0xc62a: "BaselineExposure", - 0xc62b: "BaselineNoise", - 0xc62c: "BaselineSharpness", - 0xc62d: "BayerGreenSplit", - 0xc62e: "LinearResponseLimit", - 0xc62f: "CameraSerialNumber", - 0xc630: "LensInfo", - 0xc631: "ChromaBlurRadius", - 0xc632: "AntiAliasStrength", - 0xc633: "ShadowScale", - 0xc634: "DNGPrivateData", - 0xc635: "MakerNoteSafety", - 0xc65a: "CalibrationIlluminant1", - 0xc65b: "CalibrationIlluminant2", - 0xc65c: "BestQualityScale", - 0xc65d: "RawDataUniqueID", - 0xc68b: "OriginalRawFileName", - 0xc68c: "OriginalRawFileData", - 0xc68d: "ActiveArea", - 0xc68e: "MaskedAreas", - 0xc68f: "AsShotICCProfile", - 0xc690: "AsShotPreProfileMatrix", - 0xc691: "CurrentICCProfile", - 0xc692: "CurrentPreProfileMatrix", - 0xc6bf: "ColorimetricReference", - 0xc6f3: "CameraCalibrationSignature", - 0xc6f4: "ProfileCalibrationSignature", - 0xc6f6: "AsShotProfileName", - 0xc6f7: "NoiseReductionApplied", - 0xc6f8: "ProfileName", - 0xc6f9: "ProfileHueSatMapDims", - 0xc6fa: "ProfileHueSatMapData1", - 0xc6fb: "ProfileHueSatMapData2", - 0xc6fc: "ProfileToneCurve", - 0xc6fd: "ProfileEmbedPolicy", - 0xc6fe: "ProfileCopyright", - 0xc714: "ForwardMatrix1", - 0xc715: "ForwardMatrix2", - 0xc716: "PreviewApplicationName", - 0xc717: "PreviewApplicationVersion", - 0xc718: "PreviewSettingsName", - 0xc719: "PreviewSettingsDigest", - 0xc71a: "PreviewColorSpace", - 0xc71b: "PreviewDateTime", - 0xc71c: "RawImageDigest", - 0xc71d: "OriginalRawFileDigest", - 0xc71e: "SubTileBlockSize", - 0xc71f: "RowInterleaveFactor", - 0xc725: "ProfileLookTableDims", - 0xc726: "ProfileLookTableData", - 0xc740: "OpcodeList1", - 0xc741: "OpcodeList2", - 0xc74e: "OpcodeList3", - 0xc761: "NoiseProfile" + 0x9C9B: "XPTitle", + 0x9C9C: "XPComment", + 0x9C9D: "XPAuthor", + 0x9C9E: "XPKeywords", + 0x9C9F: "XPSubject", + 0xA000: "FlashPixVersion", + 0xA001: "ColorSpace", + 0xA002: "ExifImageWidth", + 0xA003: "ExifImageHeight", + 0xA004: "RelatedSoundFile", + 0xA005: "ExifInteroperabilityOffset", + 0xA20B: "FlashEnergy", + 0xA20C: "SpatialFrequencyResponse", + 0xA20E: "FocalPlaneXResolution", + 0xA20F: "FocalPlaneYResolution", + 0xA210: "FocalPlaneResolutionUnit", + 0xA214: "SubjectLocation", + 0xA215: "ExposureIndex", + 0xA217: "SensingMethod", + 0xA300: "FileSource", + 0xA301: "SceneType", + 0xA302: "CFAPattern", + 0xA401: "CustomRendered", + 0xA402: "ExposureMode", + 0xA403: "WhiteBalance", + 0xA404: "DigitalZoomRatio", + 0xA405: "FocalLengthIn35mmFilm", + 0xA406: "SceneCaptureType", + 0xA407: "GainControl", + 0xA408: "Contrast", + 0xA409: "Saturation", + 0xA40A: "Sharpness", + 0xA40B: "DeviceSettingDescription", + 0xA40C: "SubjectDistanceRange", + 0xA420: "ImageUniqueID", + 0xA430: "CameraOwnerName", + 0xA431: "BodySerialNumber", + 0xA432: "LensSpecification", + 0xA433: "LensMake", + 0xA434: "LensModel", + 0xA435: "LensSerialNumber", + 0xA500: "Gamma", + 0xC4A5: "PrintImageMatching", + 0xC612: "DNGVersion", + 0xC613: "DNGBackwardVersion", + 0xC614: "UniqueCameraModel", + 0xC615: "LocalizedCameraModel", + 0xC616: "CFAPlaneColor", + 0xC617: "CFALayout", + 0xC618: "LinearizationTable", + 0xC619: "BlackLevelRepeatDim", + 0xC61A: "BlackLevel", + 0xC61B: "BlackLevelDeltaH", + 0xC61C: "BlackLevelDeltaV", + 0xC61D: "WhiteLevel", + 0xC61E: "DefaultScale", + 0xC61F: "DefaultCropOrigin", + 0xC620: "DefaultCropSize", + 0xC621: "ColorMatrix1", + 0xC622: "ColorMatrix2", + 0xC623: "CameraCalibration1", + 0xC624: "CameraCalibration2", + 0xC625: "ReductionMatrix1", + 0xC626: "ReductionMatrix2", + 0xC627: "AnalogBalance", + 0xC628: "AsShotNeutral", + 0xC629: "AsShotWhiteXY", + 0xC62A: "BaselineExposure", + 0xC62B: "BaselineNoise", + 0xC62C: "BaselineSharpness", + 0xC62D: "BayerGreenSplit", + 0xC62E: "LinearResponseLimit", + 0xC62F: "CameraSerialNumber", + 0xC630: "LensInfo", + 0xC631: "ChromaBlurRadius", + 0xC632: "AntiAliasStrength", + 0xC633: "ShadowScale", + 0xC634: "DNGPrivateData", + 0xC635: "MakerNoteSafety", + 0xC65A: "CalibrationIlluminant1", + 0xC65B: "CalibrationIlluminant2", + 0xC65C: "BestQualityScale", + 0xC65D: "RawDataUniqueID", + 0xC68B: "OriginalRawFileName", + 0xC68C: "OriginalRawFileData", + 0xC68D: "ActiveArea", + 0xC68E: "MaskedAreas", + 0xC68F: "AsShotICCProfile", + 0xC690: "AsShotPreProfileMatrix", + 0xC691: "CurrentICCProfile", + 0xC692: "CurrentPreProfileMatrix", + 0xC6BF: "ColorimetricReference", + 0xC6F3: "CameraCalibrationSignature", + 0xC6F4: "ProfileCalibrationSignature", + 0xC6F6: "AsShotProfileName", + 0xC6F7: "NoiseReductionApplied", + 0xC6F8: "ProfileName", + 0xC6F9: "ProfileHueSatMapDims", + 0xC6FA: "ProfileHueSatMapData1", + 0xC6FB: "ProfileHueSatMapData2", + 0xC6FC: "ProfileToneCurve", + 0xC6FD: "ProfileEmbedPolicy", + 0xC6FE: "ProfileCopyright", + 0xC714: "ForwardMatrix1", + 0xC715: "ForwardMatrix2", + 0xC716: "PreviewApplicationName", + 0xC717: "PreviewApplicationVersion", + 0xC718: "PreviewSettingsName", + 0xC719: "PreviewSettingsDigest", + 0xC71A: "PreviewColorSpace", + 0xC71B: "PreviewDateTime", + 0xC71C: "RawImageDigest", + 0xC71D: "OriginalRawFileDigest", + 0xC71E: "SubTileBlockSize", + 0xC71F: "RowInterleaveFactor", + 0xC725: "ProfileLookTableDims", + 0xC726: "ProfileLookTableData", + 0xC740: "OpcodeList1", + 0xC741: "OpcodeList2", + 0xC74E: "OpcodeList3", + 0xC761: "NoiseProfile", } ## diff --git a/server/www/packages/packages-linux/x64/PIL/FitsStubImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/FitsStubImagePlugin.py index 63c195c..7e6d35e 100644 --- a/server/www/packages/packages-linux/x64/PIL/FitsStubImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/FitsStubImagePlugin.py @@ -23,6 +23,7 @@ def register_handler(handler): global _handler _handler = handler + # -------------------------------------------------------------------- # Image adapter diff --git a/server/www/packages/packages-linux/x64/PIL/FliImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/FliImagePlugin.py index c78c8c6..82015e2 100644 --- a/server/www/packages/packages-linux/x64/PIL/FliImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/FliImagePlugin.py @@ -19,12 +19,15 @@ from . import Image, ImageFile, ImagePalette from ._binary import i8, i16le as i16, i32le as i32, o8 +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.2" # # decoder + def _accept(prefix): return len(prefix) >= 6 and i16(prefix[4:6]) in [0xAF11, 0xAF12] @@ -33,6 +36,7 @@ def _accept(prefix): # Image plugin for the FLI/FLC animation format. Use the seek # method to load individual frames. + class FliImageFile(ImageFile.ImageFile): format = "FLI" @@ -44,9 +48,11 @@ class FliImageFile(ImageFile.ImageFile): # HEAD s = self.fp.read(128) magic = i16(s[4:6]) - if not (magic in [0xAF11, 0xAF12] and - i16(s[14:16]) in [0, 3] and # flags - s[20:22] == b"\x00\x00"): # reserved + if not ( + magic in [0xAF11, 0xAF12] + and i16(s[14:16]) in [0, 3] # flags + and s[20:22] == b"\x00\x00" # reserved + ): raise SyntaxError("not an FLI/FLC file") # frames @@ -82,7 +88,7 @@ class FliImageFile(ImageFile.ImageFile): elif i16(s[4:6]) == 4: self._palette(palette, 0) - palette = [o8(r)+o8(g)+o8(b) for (r, g, b) in palette] + palette = [o8(r) + o8(g) + o8(b) for (r, g, b) in palette] self.palette = ImagePalette.raw("RGB", b"".join(palette)) # set things up to decode first frame @@ -104,8 +110,8 @@ class FliImageFile(ImageFile.ImageFile): s = self.fp.read(n * 3) for n in range(0, len(s), 3): r = i8(s[n]) << shift - g = i8(s[n+1]) << shift - b = i8(s[n+2]) << shift + g = i8(s[n + 1]) << shift + b = i8(s[n + 2]) << shift palette[i] = (r, g, b) i += 1 @@ -131,6 +137,9 @@ class FliImageFile(ImageFile.ImageFile): self.__frame = -1 self.__fp.seek(self.__rewind) self.__offset = 128 + else: + # ensure that the previous frame was loaded + self.load() if frame != self.__frame + 1: raise ValueError("cannot seek to frame %d" % frame) @@ -147,13 +156,22 @@ class FliImageFile(ImageFile.ImageFile): framesize = i32(s) self.decodermaxblock = framesize - self.tile = [("fli", (0, 0)+self.size, self.__offset, None)] + self.tile = [("fli", (0, 0) + self.size, self.__offset, None)] self.__offset += framesize def tell(self): return self.__frame + def _close__fp(self): + try: + if self.__fp != self.fp: + self.__fp.close() + except AttributeError: + pass + finally: + self.__fp = None + # # registry diff --git a/server/www/packages/packages-linux/x64/PIL/FontFile.py b/server/www/packages/packages-linux/x64/PIL/FontFile.py index 305e8af..e57c2f3 100644 --- a/server/www/packages/packages-linux/x64/PIL/FontFile.py +++ b/server/www/packages/packages-linux/x64/PIL/FontFile.py @@ -17,6 +17,7 @@ from __future__ import print_function import os + from . import Image, _binary WIDTH = 800 @@ -33,6 +34,7 @@ def puti16(fp, values): ## # Base class for raster font file handlers. + class FontFile(object): bitmap = None @@ -46,7 +48,7 @@ class FontFile(object): return self.glyph[ix] def compile(self): - "Create metrics and bitmap" + """Create metrics and bitmap""" if self.bitmap: return @@ -61,7 +63,7 @@ class FontFile(object): w = w + (src[2] - src[0]) if w > WIDTH: lines += 1 - w = (src[2] - src[0]) + w = src[2] - src[0] maxwidth = max(maxwidth, w) xsize = maxwidth @@ -93,7 +95,7 @@ class FontFile(object): self.metrics[i] = d, dst, s def save(self, filename): - "Save font" + """Save font""" self.compile() @@ -103,7 +105,7 @@ class FontFile(object): # font metrics with open(os.path.splitext(filename)[0] + ".pil", "wb") as fp: fp.write(b"PILfont\n") - fp.write((";;;;;;%d;\n" % self.ysize).encode('ascii')) # HACK!!! + fp.write((";;;;;;%d;\n" % self.ysize).encode("ascii")) # HACK!!! fp.write(b"DATA\n") for id in range(256): m = self.metrics[id] diff --git a/server/www/packages/packages-linux/x64/PIL/FpxImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/FpxImagePlugin.py index 9f284fd..15ebe0e 100644 --- a/server/www/packages/packages-linux/x64/PIL/FpxImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/FpxImagePlugin.py @@ -17,32 +17,35 @@ from __future__ import print_function -from . import Image, ImageFile -from ._binary import i32le as i32, i8 - import olefile +from . import Image, ImageFile +from ._binary import i8, i32le as i32 + +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.1" # we map from colour field tuples to (mode, rawmode) descriptors MODES = { # opacity - (0x00007ffe): ("A", "L"), + (0x00007FFE): ("A", "L"), # monochrome (0x00010000,): ("L", "L"), - (0x00018000, 0x00017ffe): ("RGBA", "LA"), + (0x00018000, 0x00017FFE): ("RGBA", "LA"), # photo YCC (0x00020000, 0x00020001, 0x00020002): ("RGB", "YCC;P"), - (0x00028000, 0x00028001, 0x00028002, 0x00027ffe): ("RGBA", "YCCA;P"), + (0x00028000, 0x00028001, 0x00028002, 0x00027FFE): ("RGBA", "YCCA;P"), # standard RGB (NIFRGB) (0x00030000, 0x00030001, 0x00030002): ("RGB", "RGB"), - (0x00038000, 0x00038001, 0x00038002, 0x00037ffe): ("RGBA", "RGBA"), + (0x00038000, 0x00038001, 0x00038002, 0x00037FFE): ("RGBA", "RGBA"), } # # -------------------------------------------------------------------- + def _accept(prefix): return prefix[:8] == olefile.MAGIC @@ -50,6 +53,7 @@ def _accept(prefix): ## # Image plugin for the FlashPix images. + class FpxImageFile(ImageFile.ImageFile): format = "FPX" @@ -74,10 +78,9 @@ class FpxImageFile(ImageFile.ImageFile): # # get the Image Contents Property Set - prop = self.ole.getproperties([ - "Data Object Store %06d" % index, - "\005Image Contents" - ]) + prop = self.ole.getproperties( + ["Data Object Store %06d" % index, "\005Image Contents"] + ) # size (highest resolution) @@ -103,7 +106,7 @@ class FpxImageFile(ImageFile.ImageFile): colors = [] for i in range(i32(s, 4)): # note: for now, we ignore the "uncalibrated" flag - colors.append(i32(s, 8+i*4) & 0x7fffffff) + colors.append(i32(s, 8 + i * 4) & 0x7FFFFFFF) self.mode, self.rawmode = MODES[tuple(colors)] @@ -123,7 +126,7 @@ class FpxImageFile(ImageFile.ImageFile): stream = [ "Data Object Store %06d" % index, "Resolution %04d" % subimage, - "Subimage 0000 Header" + "Subimage 0000 Header", ] fp = self.ole.openstream(stream) @@ -155,17 +158,29 @@ class FpxImageFile(ImageFile.ImageFile): for i in range(0, len(s), length): - compression = i32(s, i+8) + compression = i32(s, i + 8) if compression == 0: - self.tile.append(("raw", (x, y, x+xtile, y+ytile), - i32(s, i) + 28, (self.rawmode))) + self.tile.append( + ( + "raw", + (x, y, x + xtile, y + ytile), + i32(s, i) + 28, + (self.rawmode), + ) + ) elif compression == 1: # FIXME: the fill decoder is not implemented - self.tile.append(("fill", (x, y, x+xtile, y+ytile), - i32(s, i) + 28, (self.rawmode, s[12:16]))) + self.tile.append( + ( + "fill", + (x, y, x + xtile, y + ytile), + i32(s, i) + 28, + (self.rawmode, s[12:16]), + ) + ) elif compression == 2: @@ -187,8 +202,14 @@ class FpxImageFile(ImageFile.ImageFile): # The image is stored as defined by rawmode jpegmode = rawmode - self.tile.append(("jpeg", (x, y, x+xtile, y+ytile), - i32(s, i) + 28, (rawmode, jpegmode))) + self.tile.append( + ( + "jpeg", + (x, y, x + xtile, y + ytile), + i32(s, i) + 28, + (rawmode, jpegmode), + ) + ) # FIXME: jpeg tables are tile dependent; the prefix # data must be placed in the tile descriptor itself! @@ -211,11 +232,11 @@ class FpxImageFile(ImageFile.ImageFile): def load(self): if not self.fp: - self.fp = self.ole.openstream(self.stream[:2] + - ["Subimage 0000 Data"]) + self.fp = self.ole.openstream(self.stream[:2] + ["Subimage 0000 Data"]) return ImageFile.ImageFile.load(self) + # # -------------------------------------------------------------------- diff --git a/server/www/packages/packages-linux/x64/PIL/FtexImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/FtexImagePlugin.py index 08ce0e0..06f4a72 100644 --- a/server/www/packages/packages-linux/x64/PIL/FtexImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/FtexImagePlugin.py @@ -20,7 +20,14 @@ has the following structure: {format_directory} {data} Where: -{header} = { u32:magic, u32:version, u32:width, u32:height, u32:mipmap_count, u32:format_count } +{header} = { + u32:magic, + u32:version, + u32:width, + u32:height, + u32:mipmap_count, + u32:format_count +} * The "magic" number is "FTEX". * "width" and "height" are the dimensions of the texture. @@ -46,8 +53,8 @@ Note: All data is stored in little-Endian (Intel) byte order. import struct from io import BytesIO -from . import Image, ImageFile +from . import Image, ImageFile MAGIC = b"FTEX" FORMAT_DXT1 = 0 @@ -59,8 +66,8 @@ class FtexImageFile(ImageFile.ImageFile): format_description = "Texture File Format (IW2:EOC)" def _open(self): - magic = struct.unpack("= 8 and \ - i32(prefix[:4]) >= 20 and i32(prefix[4:8]) in (1, 2) + return len(prefix) >= 8 and i32(prefix[:4]) >= 20 and i32(prefix[4:8]) in (1, 2) ## # Image plugin for the GIMP brush format. + class GbrImageFile(ImageFile.ImageFile): format = "GBR" @@ -55,24 +55,23 @@ class GbrImageFile(ImageFile.ImageFile): if width <= 0 or height <= 0: raise SyntaxError("not a GIMP brush") if color_depth not in (1, 4): - raise SyntaxError( - "Unsupported GIMP brush color depth: %s" % color_depth) + raise SyntaxError("Unsupported GIMP brush color depth: %s" % color_depth) if version == 1: - comment_length = header_size-20 + comment_length = header_size - 20 else: - comment_length = header_size-28 + comment_length = header_size - 28 magic_number = self.fp.read(4) - if magic_number != b'GIMP': + if magic_number != b"GIMP": raise SyntaxError("not a GIMP brush, bad magic number") - self.info['spacing'] = i32(self.fp.read(4)) + self.info["spacing"] = i32(self.fp.read(4)) comment = self.fp.read(comment_length)[:-1] if color_depth == 1: self.mode = "L" else: - self.mode = 'RGBA' + self.mode = "RGBA" self._size = width, height @@ -88,6 +87,7 @@ class GbrImageFile(ImageFile.ImageFile): self.im = Image.core.new(self.mode, self.size) self.frombytes(self.fp.read(self._data_size)) + # # registry diff --git a/server/www/packages/packages-linux/x64/PIL/GdImageFile.py b/server/www/packages/packages-linux/x64/PIL/GdImageFile.py index 1361542..2d49235 100644 --- a/server/www/packages/packages-linux/x64/PIL/GdImageFile.py +++ b/server/www/packages/packages-linux/x64/PIL/GdImageFile.py @@ -26,6 +26,8 @@ from . import ImageFile, ImagePalette from ._binary import i8, i16be as i16, i32be as i32 +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.1" @@ -35,6 +37,7 @@ __version__ = "0.1" # this plugin, you have to import the GdImageFile module and # use the GdImageFile.open function. + class GdImageFile(ImageFile.ImageFile): format = "GD" @@ -55,14 +58,17 @@ class GdImageFile(ImageFile.ImageFile): trueColorOffset = 2 if trueColor else 0 # transparency index - tindex = i32(s[7+trueColorOffset:7+trueColorOffset+4]) + tindex = i32(s[7 + trueColorOffset : 7 + trueColorOffset + 4]) if tindex < 256: self.info["transparency"] = tindex - self.palette = ImagePalette.raw("XBGR", s[7+trueColorOffset+4:7+trueColorOffset+4+256*4]) + self.palette = ImagePalette.raw( + "XBGR", s[7 + trueColorOffset + 4 : 7 + trueColorOffset + 4 + 256 * 4] + ) - self.tile = [("raw", (0, 0)+self.size, 7+trueColorOffset+4+256*4, - ("L", 0, 1))] + self.tile = [ + ("raw", (0, 0) + self.size, 7 + trueColorOffset + 4 + 256 * 4, ("L", 0, 1)) + ] def open(fp, mode="r"): diff --git a/server/www/packages/packages-linux/x64/PIL/GifImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/GifImagePlugin.py index 107c015..9d8e96f 100644 --- a/server/www/packages/packages-linux/x64/PIL/GifImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/GifImagePlugin.py @@ -24,17 +24,20 @@ # See the README file for information on usage and redistribution. # -from . import Image, ImageFile, ImagePalette, ImageChops, ImageSequence -from ._binary import i8, i16le as i16, o8, o16le as o16 - import itertools +from . import Image, ImageChops, ImageFile, ImagePalette, ImageSequence +from ._binary import i8, i16le as i16, o8, o16le as o16 + +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.9" # -------------------------------------------------------------------- # Identify/read GIF files + def _accept(prefix): return prefix[:6] in [b"GIF87a", b"GIF89a"] @@ -43,6 +46,7 @@ def _accept(prefix): # Image plugin for GIF images. This plugin supports both GIF87 and # GIF89 images. + class GifImageFile(ImageFile.ImageFile): format = "GIF" @@ -76,7 +80,7 @@ class GifImageFile(ImageFile.ImageFile): # check if palette contains colour indices p = self.fp.read(3 << bits) for i in range(0, len(p), 3): - if not (i//3 == i8(p[i]) == i8(p[i+1]) == i8(p[i+2])): + if not (i // 3 == i8(p[i]) == i8(p[i + 1]) == i8(p[i + 2])): p = ImagePalette.raw("RGB", p) self.global_palette = self.palette = p break @@ -120,6 +124,8 @@ class GifImageFile(ImageFile.ImageFile): if not self._seek_check(frame): return if frame < self.__frame: + if frame != 0: + self.im = None self._seek(0) last_frame = self.__frame @@ -164,6 +170,7 @@ class GifImageFile(ImageFile.ImageFile): self.im.paste(self.dispose, self.dispose_extent) from copy import copy + self.palette = copy(self.global_palette) info = {} @@ -201,7 +208,13 @@ class GifImageFile(ImageFile.ImageFile): # # comment extension # - info["comment"] = block + while block: + if "comment" in info: + info["comment"] += block + else: + info["comment"] = block + block = self.data() + continue elif i8(s) == 255: # # application extension @@ -223,6 +236,8 @@ class GifImageFile(ImageFile.ImageFile): # extent x0, y0 = i16(s[0:]), i16(s[2:]) x1, y1 = x0 + i16(s[4:]), y0 + i16(s[6:]) + if x1 > self.size[0] or y1 > self.size[1]: + self._size = max(x1, self.size[0]), max(y1, self.size[1]) self.dispose_extent = x0, y0, x1, y1 flags = i8(s[8]) @@ -230,16 +245,14 @@ class GifImageFile(ImageFile.ImageFile): if flags & 128: bits = (flags & 7) + 1 - self.palette =\ - ImagePalette.raw("RGB", self.fp.read(3 << bits)) + self.palette = ImagePalette.raw("RGB", self.fp.read(3 << bits)) # image data bits = i8(self.fp.read(1)) self.__offset = self.fp.tell() - self.tile = [("gif", - (x0, y0, x1, y1), - self.__offset, - (bits, interlace))] + self.tile = [ + ("gif", (x0, y0, x1, y1), self.__offset, (bits, interlace)) + ] break else: @@ -252,8 +265,8 @@ class GifImageFile(ImageFile.ImageFile): self.dispose = None elif self.disposal_method == 2: # replace with background colour - self.dispose = Image.core.fill("P", self.size, - self.info["background"]) + Image._decompression_bomb_check(self.size) + self.dispose = Image.core.fill("P", self.size, self.info["background"]) else: # replace with previous contents if self.im: @@ -291,20 +304,25 @@ class GifImageFile(ImageFile.ImageFile): # we do this by pasting the updated area onto the previous # frame which we then use as the current image content updated = self._crop(self.im, self.dispose_extent) - self._prev_im.paste(updated, self.dispose_extent, - updated.convert('RGBA')) + self._prev_im.paste(updated, self.dispose_extent, updated.convert("RGBA")) self.im = self._prev_im self._prev_im = self.im.copy() + def _close__fp(self): + try: + if self.__fp != self.fp: + self.__fp.close() + except AttributeError: + pass + finally: + self.__fp = None + + # -------------------------------------------------------------------- # Write GIF files -RAWMODE = { - "1": "L", - "L": "L", - "P": "P" -} +RAWMODE = {"1": "L", "L": "L", "P": "P"} def _normalize_mode(im, initial_call=False): @@ -355,19 +373,23 @@ def _normalize_palette(im, palette, info): if isinstance(palette, (bytes, bytearray, list)): source_palette = bytearray(palette[:768]) if isinstance(palette, ImagePalette.ImagePalette): - source_palette = bytearray(itertools.chain.from_iterable( - zip(palette.palette[:256], - palette.palette[256:512], - palette.palette[512:768]))) + source_palette = bytearray( + itertools.chain.from_iterable( + zip( + palette.palette[:256], + palette.palette[256:512], + palette.palette[512:768], + ) + ) + ) if im.mode == "P": if not source_palette: source_palette = im.im.getpalette("RGB")[:768] else: # L-mode if not source_palette: - source_palette = bytearray(i//3 for i in range(768)) - im.palette = ImagePalette.ImagePalette("RGB", - palette=source_palette) + source_palette = bytearray(i // 3 for i in range(768)) + im.palette = ImagePalette.ImagePalette("RGB", palette=source_palette) used_palette_colors = _get_optimize(im, info) if used_palette_colors is not None: @@ -379,6 +401,8 @@ def _normalize_palette(im, palette, info): def _write_single_frame(im, fp, palette): im_out = _normalize_mode(im, True) + for k, v in im_out.info.items(): + im.encoderinfo.setdefault(k, v) im_out = _normalize_palette(im_out, palette, im.encoderinfo) for s in _get_global_header(im_out, im.encoderinfo): @@ -391,29 +415,31 @@ def _write_single_frame(im, fp, palette): _write_local_header(fp, im, (0, 0), flags) im_out.encoderconfig = (8, get_interlace(im)) - ImageFile._save(im_out, fp, [("gif", (0, 0)+im.size, 0, - RAWMODE[im_out.mode])]) + ImageFile._save(im_out, fp, [("gif", (0, 0) + im.size, 0, RAWMODE[im_out.mode])]) fp.write(b"\0") # end of image data def _write_multiple_frames(im, fp, palette): - duration = im.encoderinfo.get("duration", None) - disposal = im.encoderinfo.get('disposal', None) + duration = im.encoderinfo.get("duration", im.info.get("duration")) + disposal = im.encoderinfo.get("disposal", im.info.get("disposal")) im_frames = [] frame_count = 0 - for imSequence in itertools.chain([im], - im.encoderinfo.get("append_images", [])): + background_im = None + for imSequence in itertools.chain([im], im.encoderinfo.get("append_images", [])): for im_frame in ImageSequence.Iterator(imSequence): # a copy is required here since seek can still mutate the image im_frame = _normalize_mode(im_frame.copy()) + if frame_count == 0: + for k, v in im_frame.info.items(): + im.encoderinfo.setdefault(k, v) im_frame = _normalize_palette(im_frame, palette, im.encoderinfo) encoderinfo = im.encoderinfo.copy() if isinstance(duration, (list, tuple)): - encoderinfo['duration'] = duration[frame_count] + encoderinfo["duration"] = duration[frame_count] if isinstance(disposal, (list, tuple)): encoderinfo["disposal"] = disposal[frame_count] frame_count += 1 @@ -421,45 +447,54 @@ def _write_multiple_frames(im, fp, palette): if im_frames: # delta frame previous = im_frames[-1] - if _get_palette_bytes(im_frame) == \ - _get_palette_bytes(previous['im']): - delta = ImageChops.subtract_modulo(im_frame, - previous['im']) + if encoderinfo.get("disposal") == 2: + if background_im is None: + background = _get_background( + im, + im.encoderinfo.get("background", im.info.get("background")), + ) + background_im = Image.new("P", im_frame.size, background) + background_im.putpalette(im_frames[0]["im"].palette) + base_im = background_im + else: + base_im = previous["im"] + if _get_palette_bytes(im_frame) == _get_palette_bytes(base_im): + delta = ImageChops.subtract_modulo(im_frame, base_im) else: delta = ImageChops.subtract_modulo( - im_frame.convert('RGB'), previous['im'].convert('RGB')) + im_frame.convert("RGB"), base_im.convert("RGB") + ) bbox = delta.getbbox() if not bbox: # This frame is identical to the previous frame if duration: - previous['encoderinfo']['duration'] += \ - encoderinfo['duration'] + previous["encoderinfo"]["duration"] += encoderinfo["duration"] continue else: bbox = None - im_frames.append({ - 'im': im_frame, - 'bbox': bbox, - 'encoderinfo': encoderinfo - }) + im_frames.append({"im": im_frame, "bbox": bbox, "encoderinfo": encoderinfo}) if len(im_frames) > 1: for frame_data in im_frames: - im_frame = frame_data['im'] - if not frame_data['bbox']: + im_frame = frame_data["im"] + if not frame_data["bbox"]: # global header - for s in _get_global_header(im_frame, - frame_data['encoderinfo']): + for s in _get_global_header(im_frame, frame_data["encoderinfo"]): fp.write(s) offset = (0, 0) else: # compress difference - frame_data['encoderinfo']['include_color_table'] = True + frame_data["encoderinfo"]["include_color_table"] = True - im_frame = im_frame.crop(frame_data['bbox']) - offset = frame_data['bbox'][:2] - _write_frame_data(fp, im_frame, offset, frame_data['encoderinfo']) + im_frame = im_frame.crop(frame_data["bbox"]) + offset = frame_data["bbox"][:2] + _write_frame_data(fp, im_frame, offset, frame_data["encoderinfo"]) return True + elif "duration" in im.encoderinfo and isinstance( + im.encoderinfo["duration"], (list, tuple) + ): + # Since multiple frames will not be written, add together the frame durations + im.encoderinfo["duration"] = sum(im.encoderinfo["duration"]) def _save_all(im, fp, filename): @@ -467,12 +502,10 @@ def _save_all(im, fp, filename): def _save(im, fp, filename, save_all=False): - for k, v in im.info.items(): - im.encoderinfo.setdefault(k, v) # header - try: - palette = im.encoderinfo["palette"] - except KeyError: + if "palette" in im.encoderinfo or "palette" in im.info: + palette = im.encoderinfo.get("palette", im.info.get("palette")) + else: palette = None im.encoderinfo["optimize"] = im.encoderinfo.get("optimize", True) @@ -519,7 +552,7 @@ def _write_local_header(fp, im, offset, flags): else: duration = 0 - disposal = int(im.encoderinfo.get('disposal', 0)) + disposal = int(im.encoderinfo.get("disposal", 0)) if transparent_color_exists or duration != 0 or disposal: packed_flag = 1 if transparent_color_exists else 0 @@ -527,48 +560,53 @@ def _write_local_header(fp, im, offset, flags): if not transparent_color_exists: transparency = 0 - fp.write(b"!" + - o8(249) + # extension intro - o8(4) + # length - o8(packed_flag) + # packed fields - o16(duration) + # duration - o8(transparency) + # transparency index - o8(0)) + fp.write( + b"!" + + o8(249) # extension intro + + o8(4) # length + + o8(packed_flag) # packed fields + + o16(duration) # duration + + o8(transparency) # transparency index + + o8(0) + ) - if "comment" in im.encoderinfo and \ - 1 <= len(im.encoderinfo["comment"]) <= 255: - fp.write(b"!" + - o8(254) + # extension intro - o8(len(im.encoderinfo["comment"])) + - im.encoderinfo["comment"] + - o8(0)) + if "comment" in im.encoderinfo and 1 <= len(im.encoderinfo["comment"]): + fp.write(b"!" + o8(254)) # extension intro + for i in range(0, len(im.encoderinfo["comment"]), 255): + subblock = im.encoderinfo["comment"][i : i + 255] + fp.write(o8(len(subblock)) + subblock) + fp.write(o8(0)) if "loop" in im.encoderinfo: number_of_loops = im.encoderinfo["loop"] - fp.write(b"!" + - o8(255) + # extension intro - o8(11) + - b"NETSCAPE2.0" + - o8(3) + - o8(1) + - o16(number_of_loops) + # number of loops - o8(0)) - include_color_table = im.encoderinfo.get('include_color_table') + fp.write( + b"!" + + o8(255) # extension intro + + o8(11) + + b"NETSCAPE2.0" + + o8(3) + + o8(1) + + o16(number_of_loops) # number of loops + + o8(0) + ) + include_color_table = im.encoderinfo.get("include_color_table") if include_color_table: palette_bytes = _get_palette_bytes(im) color_table_size = _get_color_table_size(palette_bytes) if color_table_size: - flags = flags | 128 # local color table flag + flags = flags | 128 # local color table flag flags = flags | color_table_size - fp.write(b"," + - o16(offset[0]) + # offset - o16(offset[1]) + - o16(im.size[0]) + # size - o16(im.size[1]) + - o8(flags)) # flags + fp.write( + b"," + + o16(offset[0]) # offset + + o16(offset[1]) + + o16(im.size[0]) # size + + o16(im.size[1]) + + o8(flags) # flags + ) if include_color_table and color_table_size: fp.write(_get_header_palette(palette_bytes)) - fp.write(o8(8)) # bits + fp.write(o8(8)) # bits def _save_netpbm(im, fp, filename): @@ -582,21 +620,23 @@ def _save_netpbm(im, fp, filename): import os from subprocess import Popen, check_call, PIPE, CalledProcessError - file = im._dump() - with open(filename, 'wb') as f: + tempfile = im._dump() + + with open(filename, "wb") as f: if im.mode != "RGB": - with open(os.devnull, 'wb') as devnull: - check_call(["ppmtogif", file], stdout=f, stderr=devnull) + with open(os.devnull, "wb") as devnull: + check_call(["ppmtogif", tempfile], stdout=f, stderr=devnull) else: # Pipe ppmquant output into ppmtogif - # "ppmquant 256 %s | ppmtogif > %s" % (file, filename) - quant_cmd = ["ppmquant", "256", file] + # "ppmquant 256 %s | ppmtogif > %s" % (tempfile, filename) + quant_cmd = ["ppmquant", "256", tempfile] togif_cmd = ["ppmtogif"] - with open(os.devnull, 'wb') as devnull: + with open(os.devnull, "wb") as devnull: quant_proc = Popen(quant_cmd, stdout=PIPE, stderr=devnull) - togif_proc = Popen(togif_cmd, stdin=quant_proc.stdout, - stdout=f, stderr=devnull) + togif_proc = Popen( + togif_cmd, stdin=quant_proc.stdout, stdout=f, stderr=devnull + ) # Allow ppmquant to receive SIGPIPE if ppmtogif exits quant_proc.stdout.close() @@ -610,7 +650,7 @@ def _save_netpbm(im, fp, filename): raise CalledProcessError(retcode, togif_cmd) try: - os.unlink(file) + os.unlink(tempfile) except OSError: pass @@ -642,7 +682,7 @@ def _get_optimize(im, info): # * If we have a 'large' image, the palette is in the noise. # create the new palette if not every color is used - optimise = _FORCE_OPTIMIZE or im.mode == 'L' + optimise = _FORCE_OPTIMIZE or im.mode == "L" if optimise or im.width * im.height < 512 * 512: # check which colors are used used_palette_colors = [] @@ -650,18 +690,23 @@ def _get_optimize(im, info): if count: used_palette_colors.append(i) - if optimise or (len(used_palette_colors) <= 128 and - max(used_palette_colors) > len(used_palette_colors)): + if optimise or ( + len(used_palette_colors) <= 128 + and max(used_palette_colors) > len(used_palette_colors) + ): return used_palette_colors def _get_color_table_size(palette_bytes): # calculate the palette size for the header import math - color_table_size = int(math.ceil(math.log(len(palette_bytes)//3, 2)))-1 - if color_table_size < 0: - color_table_size = 0 - return color_table_size + + if not palette_bytes: + return 0 + elif len(palette_bytes) < 9: + return 1 + else: + return int(math.ceil(math.log(len(palette_bytes) // 3, 2))) - 1 def _get_header_palette(palette_bytes): @@ -676,7 +721,7 @@ def _get_header_palette(palette_bytes): # add the missing amount of bytes # the palette has to be 2< 0: palette_bytes += o8(0) * 3 * actual_target_size_diff return palette_bytes @@ -692,6 +737,18 @@ def _get_palette_bytes(im): return im.palette.palette +def _get_background(im, infoBackground): + background = 0 + if infoBackground: + background = infoBackground + if isinstance(background, tuple): + # WebPImagePlugin stores an RGBA value in info["background"] + # So it must be converted to the same format as GifImagePlugin's + # info["background"] - a global color table index + background = im.palette.getcolor(background) + return background + + def _get_global_header(im, info): """Return a list of strings representing a GIF header""" @@ -701,9 +758,9 @@ def _get_global_header(im, info): version = b"87a" for extensionKey in ["transparency", "duration", "loop", "comment"]: if info and extensionKey in info: - if ((extensionKey == "duration" and info[extensionKey] == 0) or - (extensionKey == "comment" and - not (1 <= len(info[extensionKey]) <= 255))): + if (extensionKey == "duration" and info[extensionKey] == 0) or ( + extensionKey == "comment" and not (1 <= len(info[extensionKey]) <= 255) + ): continue version = b"89a" break @@ -711,24 +768,23 @@ def _get_global_header(im, info): if im.info.get("version") == b"89a": version = b"89a" + background = _get_background(im, info.get("background")) + palette_bytes = _get_palette_bytes(im) color_table_size = _get_color_table_size(palette_bytes) - background = info["background"] if "background" in info else 0 - return [ - b"GIF"+version + # signature + version - o16(im.size[0]) + # canvas width - o16(im.size[1]), # canvas height - + b"GIF" # signature + + version # version + + o16(im.size[0]) # canvas width + + o16(im.size[1]), # canvas height # Logical Screen Descriptor # size of global color table + global color table flag - o8(color_table_size + 128), # packed fields + o8(color_table_size + 128), # packed fields # background + reserved/aspect o8(background) + o8(0), - # Global Color Table - _get_header_palette(palette_bytes) + _get_header_palette(palette_bytes), ] @@ -739,13 +795,15 @@ def _write_frame_data(fp, im_frame, offset, params): # local image header _write_local_header(fp, im_frame, offset, 0) - ImageFile._save(im_frame, fp, [("gif", (0, 0)+im_frame.size, 0, - RAWMODE[im_frame.mode])]) + ImageFile._save( + im_frame, fp, [("gif", (0, 0) + im_frame.size, 0, RAWMODE[im_frame.mode])] + ) fp.write(b"\0") # end of image data finally: del im_frame.encoderinfo + # -------------------------------------------------------------------- # Legacy GIF utilities @@ -794,6 +852,7 @@ def getdata(im, offset=(0, 0), **params): :returns: List of Bytes containing gif encoded frame data """ + class Collector(object): data = [] diff --git a/server/www/packages/packages-linux/x64/PIL/GimpGradientFile.py b/server/www/packages/packages-linux/x64/PIL/GimpGradientFile.py index 10593da..f48e7f7 100644 --- a/server/www/packages/packages-linux/x64/PIL/GimpGradientFile.py +++ b/server/www/packages/packages-linux/x64/PIL/GimpGradientFile.py @@ -13,7 +13,8 @@ # See the README file for information on usage and redistribution. # -from math import pi, log, sin, sqrt +from math import log, pi, sin, sqrt + from ._binary import o8 # -------------------------------------------------------------------- @@ -72,7 +73,7 @@ class GradientFile(object): for i in range(entries): - x = i / float(entries-1) + x = i / float(entries - 1) while x1 < x: ix += 1 @@ -100,8 +101,8 @@ class GradientFile(object): ## # File handler for GIMP's gradient format. -class GimpGradientFile(GradientFile): +class GimpGradientFile(GradientFile): def __init__(self, fp): if fp.readline()[:13] != b"GIMP Gradient": diff --git a/server/www/packages/packages-linux/x64/PIL/GimpPaletteFile.py b/server/www/packages/packages-linux/x64/PIL/GimpPaletteFile.py index 6eef6a2..2994bbe 100644 --- a/server/www/packages/packages-linux/x64/PIL/GimpPaletteFile.py +++ b/server/www/packages/packages-linux/x64/PIL/GimpPaletteFile.py @@ -15,31 +15,30 @@ # import re -from ._binary import o8 +from ._binary import o8 ## # File handler for GIMP's palette format. + class GimpPaletteFile(object): rawmode = "RGB" def __init__(self, fp): - self.palette = [o8(i)*3 for i in range(256)] + self.palette = [o8(i) * 3 for i in range(256)] if fp.readline()[:12] != b"GIMP Palette": raise SyntaxError("not a GIMP palette file") - i = 0 - - while i <= 255: + for i in range(256): s = fp.readline() - if not s: break + # skip fields and comment lines if re.match(br"\w+:|#", s): continue @@ -50,10 +49,7 @@ class GimpPaletteFile(object): if len(v) != 3: raise ValueError("bad palette entry") - if 0 <= i <= 255: - self.palette[i] = o8(v[0]) + o8(v[1]) + o8(v[2]) - - i += 1 + self.palette[i] = o8(v[0]) + o8(v[1]) + o8(v[2]) self.palette = b"".join(self.palette) diff --git a/server/www/packages/packages-linux/x64/PIL/GribStubImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/GribStubImagePlugin.py index 243ea2a..8a24a98 100644 --- a/server/www/packages/packages-linux/x64/PIL/GribStubImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/GribStubImagePlugin.py @@ -28,6 +28,7 @@ def register_handler(handler): # -------------------------------------------------------------------- # Image adapter + def _accept(prefix): return prefix[0:4] == b"GRIB" and i8(prefix[7]) == 1 diff --git a/server/www/packages/packages-linux/x64/PIL/Hdf5StubImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/Hdf5StubImagePlugin.py index 8783f80..a3ea12f 100644 --- a/server/www/packages/packages-linux/x64/PIL/Hdf5StubImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/Hdf5StubImagePlugin.py @@ -27,6 +27,7 @@ def register_handler(handler): # -------------------------------------------------------------------- # Image adapter + def _accept(prefix): return prefix[:8] == b"\x89HDF\r\n\x1a\n" diff --git a/server/www/packages/packages-linux/x64/PIL/IcnsImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/IcnsImagePlugin.py index 21236d4..75ea18b 100644 --- a/server/www/packages/packages-linux/x64/PIL/IcnsImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/IcnsImagePlugin.py @@ -15,8 +15,6 @@ # See the README file for information on usage and redistribution. # -from PIL import Image, ImageFile, PngImagePlugin -from PIL._binary import i8 import io import os import shutil @@ -24,7 +22,10 @@ import struct import sys import tempfile -enable_jpeg2k = hasattr(Image.core, 'jp2klib_version') +from PIL import Image, ImageFile, PngImagePlugin +from PIL._binary import i8 + +enable_jpeg2k = hasattr(Image.core, "jp2klib_version") if enable_jpeg2k: from PIL import Jpeg2KImagePlugin @@ -32,7 +33,7 @@ HEADERSIZE = 8 def nextheader(fobj): - return struct.unpack('>4sI', fobj.read(HEADERSIZE)) + return struct.unpack(">4sI", fobj.read(HEADERSIZE)) def read_32t(fobj, start_length, size): @@ -40,8 +41,8 @@ def read_32t(fobj, start_length, size): (start, length) = start_length fobj.seek(start) sig = fobj.read(4) - if sig != b'\x00\x00\x00\x00': - raise SyntaxError('Unknown signature, expecting 0x00000000') + if sig != b"\x00\x00\x00\x00": + raise SyntaxError("Unknown signature, expecting 0x00000000") return read_32(fobj, (start + 4, length - 4), size) @@ -81,12 +82,8 @@ def read_32(fobj, start_length, size): if bytesleft <= 0: break if bytesleft != 0: - raise SyntaxError( - "Error reading channel [%r left]" % bytesleft - ) - band = Image.frombuffer( - "L", pixel_size, b"".join(data), "raw", "L", 0, 1 - ) + raise SyntaxError("Error reading channel [%r left]" % bytesleft) + band = Image.frombuffer("L", pixel_size, b"".join(data), "raw", "L", 0, 1) im.im.putband(band.im, band_ix) return {"RGB": im} @@ -97,9 +94,7 @@ def read_mk(fobj, start_length, size): fobj.seek(start) pixel_size = (size[0] * size[2], size[1] * size[2]) sizesq = pixel_size[0] * pixel_size[1] - band = Image.frombuffer( - "L", pixel_size, fobj.read(sizesq), "raw", "L", 0, 1 - ) + band = Image.frombuffer("L", pixel_size, fobj.read(sizesq), "raw", "L", 0, 1) return {"A": band} @@ -107,73 +102,58 @@ def read_png_or_jpeg2000(fobj, start_length, size): (start, length) = start_length fobj.seek(start) sig = fobj.read(12) - if sig[:8] == b'\x89PNG\x0d\x0a\x1a\x0a': + if sig[:8] == b"\x89PNG\x0d\x0a\x1a\x0a": fobj.seek(start) im = PngImagePlugin.PngImageFile(fobj) return {"RGBA": im} - elif sig[:4] == b'\xff\x4f\xff\x51' \ - or sig[:4] == b'\x0d\x0a\x87\x0a' \ - or sig == b'\x00\x00\x00\x0cjP \x0d\x0a\x87\x0a': + elif ( + sig[:4] == b"\xff\x4f\xff\x51" + or sig[:4] == b"\x0d\x0a\x87\x0a" + or sig == b"\x00\x00\x00\x0cjP \x0d\x0a\x87\x0a" + ): if not enable_jpeg2k: - raise ValueError('Unsupported icon subimage format (rebuild PIL ' - 'with JPEG 2000 support to fix this)') + raise ValueError( + "Unsupported icon subimage format (rebuild PIL " + "with JPEG 2000 support to fix this)" + ) # j2k, jpc or j2c fobj.seek(start) jp2kstream = fobj.read(length) f = io.BytesIO(jp2kstream) im = Jpeg2KImagePlugin.Jpeg2KImageFile(f) - if im.mode != 'RGBA': - im = im.convert('RGBA') + if im.mode != "RGBA": + im = im.convert("RGBA") return {"RGBA": im} else: - raise ValueError('Unsupported icon subimage format') + raise ValueError("Unsupported icon subimage format") class IcnsFile(object): SIZES = { - (512, 512, 2): [ - (b'ic10', read_png_or_jpeg2000), - ], - (512, 512, 1): [ - (b'ic09', read_png_or_jpeg2000), - ], - (256, 256, 2): [ - (b'ic14', read_png_or_jpeg2000), - ], - (256, 256, 1): [ - (b'ic08', read_png_or_jpeg2000), - ], - (128, 128, 2): [ - (b'ic13', read_png_or_jpeg2000), - ], + (512, 512, 2): [(b"ic10", read_png_or_jpeg2000)], + (512, 512, 1): [(b"ic09", read_png_or_jpeg2000)], + (256, 256, 2): [(b"ic14", read_png_or_jpeg2000)], + (256, 256, 1): [(b"ic08", read_png_or_jpeg2000)], + (128, 128, 2): [(b"ic13", read_png_or_jpeg2000)], (128, 128, 1): [ - (b'ic07', read_png_or_jpeg2000), - (b'it32', read_32t), - (b't8mk', read_mk), - ], - (64, 64, 1): [ - (b'icp6', read_png_or_jpeg2000), - ], - (32, 32, 2): [ - (b'ic12', read_png_or_jpeg2000), - ], - (48, 48, 1): [ - (b'ih32', read_32), - (b'h8mk', read_mk), + (b"ic07", read_png_or_jpeg2000), + (b"it32", read_32t), + (b"t8mk", read_mk), ], + (64, 64, 1): [(b"icp6", read_png_or_jpeg2000)], + (32, 32, 2): [(b"ic12", read_png_or_jpeg2000)], + (48, 48, 1): [(b"ih32", read_32), (b"h8mk", read_mk)], (32, 32, 1): [ - (b'icp5', read_png_or_jpeg2000), - (b'il32', read_32), - (b'l8mk', read_mk), - ], - (16, 16, 2): [ - (b'ic11', read_png_or_jpeg2000), + (b"icp5", read_png_or_jpeg2000), + (b"il32", read_32), + (b"l8mk", read_mk), ], + (16, 16, 2): [(b"ic11", read_png_or_jpeg2000)], (16, 16, 1): [ - (b'icp4', read_png_or_jpeg2000), - (b'is32', read_32), - (b's8mk', read_mk), + (b"icp4", read_png_or_jpeg2000), + (b"is32", read_32), + (b"s8mk", read_mk), ], } @@ -185,17 +165,17 @@ class IcnsFile(object): self.dct = dct = {} self.fobj = fobj sig, filesize = nextheader(fobj) - if sig != b'icns': - raise SyntaxError('not an icns file') + if sig != b"icns": + raise SyntaxError("not an icns file") i = HEADERSIZE while i < filesize: sig, blocksize = nextheader(fobj) if blocksize <= 0: - raise SyntaxError('invalid block header') + raise SyntaxError("invalid block header") i += HEADERSIZE blocksize -= HEADERSIZE dct[sig] = (i, blocksize) - fobj.seek(blocksize, 1) + fobj.seek(blocksize, io.SEEK_CUR) i += blocksize def itersizes(self): @@ -233,7 +213,7 @@ class IcnsFile(object): size = (size[0], size[1], 1) channels = self.dataforsize(size) - im = channels.get('RGBA', None) + im = channels.get("RGBA", None) if im: return im @@ -248,6 +228,7 @@ class IcnsFile(object): ## # Image plugin for Mac OS icons. + class IcnsImageFile(ImageFile.ImageFile): """ PIL image support for Mac OS .icns files. @@ -264,13 +245,13 @@ class IcnsImageFile(ImageFile.ImageFile): def _open(self): self.icns = IcnsFile(self.fp) - self.mode = 'RGBA' - self.info['sizes'] = self.icns.itersizes() + self.mode = "RGBA" + self.info["sizes"] = self.icns.itersizes() self.best_size = self.icns.bestsize() - self.size = (self.best_size[0] * self.best_size[2], - self.best_size[1] * self.best_size[2]) - # Just use this to see if it's loaded or not yet. - self.tile = ('',) + self.size = ( + self.best_size[0] * self.best_size[2], + self.best_size[1] * self.best_size[2], + ) @property def size(self): @@ -279,27 +260,33 @@ class IcnsImageFile(ImageFile.ImageFile): @size.setter def size(self, value): info_size = value - if info_size not in self.info['sizes'] and len(info_size) == 2: + if info_size not in self.info["sizes"] and len(info_size) == 2: info_size = (info_size[0], info_size[1], 1) - if info_size not in self.info['sizes'] and len(info_size) == 3 and \ - info_size[2] == 1: - simple_sizes = [(size[0] * size[2], size[1] * size[2]) - for size in self.info['sizes']] + if ( + info_size not in self.info["sizes"] + and len(info_size) == 3 + and info_size[2] == 1 + ): + simple_sizes = [ + (size[0] * size[2], size[1] * size[2]) for size in self.info["sizes"] + ] if value in simple_sizes: - info_size = self.info['sizes'][simple_sizes.index(value)] - if info_size not in self.info['sizes']: - raise ValueError( - "This is not one of the allowed sizes of this image") + info_size = self.info["sizes"][simple_sizes.index(value)] + if info_size not in self.info["sizes"]: + raise ValueError("This is not one of the allowed sizes of this image") self._size = value def load(self): if len(self.size) == 3: self.best_size = self.size - self.size = (self.best_size[0] * self.best_size[2], - self.best_size[1] * self.best_size[2]) + self.size = ( + self.best_size[0] * self.best_size[2], + self.best_size[1] * self.best_size[2], + ) Image.Image.load(self) - if not self.tile: + if self.im and self.im.size == self.size: + # Already loaded return self.load_prepare() # This is likely NOT the best way to do it, but whatever. @@ -311,9 +298,6 @@ class IcnsImageFile(ImageFile.ImageFile): self.im = im.im self.mode = im.mode self.size = im.size - self.fp = None - self.icns = None - self.tile = () self.load_end() @@ -329,30 +313,30 @@ def _save(im, fp, filename): fp.flush() # create the temporary set of pngs - iconset = tempfile.mkdtemp('.iconset') - provided_images = {im.width: im - for im in im.encoderinfo.get("append_images", [])} + iconset = tempfile.mkdtemp(".iconset") + provided_images = {im.width: im for im in im.encoderinfo.get("append_images", [])} last_w = None + second_path = None for w in [16, 32, 128, 256, 512]: - prefix = 'icon_{}x{}'.format(w, w) + prefix = "icon_{}x{}".format(w, w) - first_path = os.path.join(iconset, prefix+'.png') + first_path = os.path.join(iconset, prefix + ".png") if last_w == w: shutil.copyfile(second_path, first_path) else: im_w = provided_images.get(w, im.resize((w, w), Image.LANCZOS)) im_w.save(first_path) - second_path = os.path.join(iconset, prefix+'@2x.png') - im_w2 = provided_images.get(w*2, im.resize((w*2, w*2), Image.LANCZOS)) + second_path = os.path.join(iconset, prefix + "@2x.png") + im_w2 = provided_images.get(w * 2, im.resize((w * 2, w * 2), Image.LANCZOS)) im_w2.save(second_path) - last_w = w*2 + last_w = w * 2 # iconutil -c icns -o {} {} from subprocess import Popen, PIPE, CalledProcessError convert_cmd = ["iconutil", "-c", "icns", "-o", filename, iconset] - with open(os.devnull, 'wb') as devnull: + with open(os.devnull, "wb") as devnull: convert_proc = Popen(convert_cmd, stdout=PIPE, stderr=devnull) convert_proc.stdout.close() @@ -366,29 +350,28 @@ def _save(im, fp, filename): raise CalledProcessError(retcode, convert_cmd) -Image.register_open(IcnsImageFile.format, IcnsImageFile, - lambda x: x[:4] == b'icns') -Image.register_extension(IcnsImageFile.format, '.icns') +Image.register_open(IcnsImageFile.format, IcnsImageFile, lambda x: x[:4] == b"icns") +Image.register_extension(IcnsImageFile.format, ".icns") -if sys.platform == 'darwin': +if sys.platform == "darwin": Image.register_save(IcnsImageFile.format, _save) Image.register_mime(IcnsImageFile.format, "image/icns") -if __name__ == '__main__': +if __name__ == "__main__": if len(sys.argv) < 2: print("Syntax: python IcnsImagePlugin.py [file]") sys.exit() - imf = IcnsImageFile(open(sys.argv[1], 'rb')) - for size in imf.info['sizes']: + imf = IcnsImageFile(open(sys.argv[1], "rb")) + for size in imf.info["sizes"]: imf.size = size imf.load() im = imf.im - im.save('out-%s-%s-%s.png' % size) + im.save("out-%s-%s-%s.png" % size) im = Image.open(sys.argv[1]) im.save("out.png") - if sys.platform == 'windows': + if sys.platform == "windows": os.startfile("out.png") diff --git a/server/www/packages/packages-linux/x64/PIL/IcoImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/IcoImagePlugin.py index 589ef3c..148e604 100644 --- a/server/www/packages/packages-linux/x64/PIL/IcoImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/IcoImagePlugin.py @@ -23,12 +23,15 @@ import struct +import warnings from io import BytesIO +from math import ceil, log -from . import Image, ImageFile, BmpImagePlugin, PngImagePlugin +from . import BmpImagePlugin, Image, ImageFile, PngImagePlugin from ._binary import i8, i16le as i16, i32le as i32 -from math import log, ceil +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.1" # @@ -39,16 +42,20 @@ _MAGIC = b"\0\0\1\0" def _save(im, fp, filename): fp.write(_MAGIC) # (2+2) - sizes = im.encoderinfo.get("sizes", - [(16, 16), (24, 24), (32, 32), (48, 48), - (64, 64), (128, 128), (256, 256)]) + sizes = im.encoderinfo.get( + "sizes", + [(16, 16), (24, 24), (32, 32), (48, 48), (64, 64), (128, 128), (256, 256)], + ) width, height = im.size - sizes = filter(lambda x: False if (x[0] > width or x[1] > height or - x[0] > 256 or x[1] > 256) else True, - sizes) + sizes = filter( + lambda x: False + if (x[0] > width or x[1] > height or x[0] > 256 or x[1] > 256) + else True, + sizes, + ) sizes = list(sizes) fp.write(struct.pack("=8bpp) - 'reserved': i8(s[3]), - 'planes': i16(s[4:]), - 'bpp': i16(s[6:]), - 'size': i32(s[8:]), - 'offset': i32(s[12:]) + "width": i8(s[0]), + "height": i8(s[1]), + "nb_color": i8(s[2]), # No. of colors in image (0 if >=8bpp) + "reserved": i8(s[3]), + "planes": i16(s[4:]), + "bpp": i16(s[6:]), + "size": i32(s[8:]), + "offset": i32(s[12:]), } # See Wikipedia - for j in ('width', 'height'): + for j in ("width", "height"): if not icon_header[j]: icon_header[j] = 256 # See Wikipedia notes about color depth. # We need this just to differ images with equal sizes - icon_header['color_depth'] = (icon_header['bpp'] or - (icon_header['nb_color'] != 0 and - ceil(log(icon_header['nb_color'], - 2))) or 256) + icon_header["color_depth"] = ( + icon_header["bpp"] + or ( + icon_header["nb_color"] != 0 + and ceil(log(icon_header["nb_color"], 2)) + ) + or 256 + ) - icon_header['dim'] = (icon_header['width'], icon_header['height']) - icon_header['square'] = (icon_header['width'] * - icon_header['height']) + icon_header["dim"] = (icon_header["width"], icon_header["height"]) + icon_header["square"] = icon_header["width"] * icon_header["height"] self.entry.append(icon_header) - self.entry = sorted(self.entry, key=lambda x: x['color_depth']) + self.entry = sorted(self.entry, key=lambda x: x["color_depth"]) # ICO images are usually squares # self.entry = sorted(self.entry, key=lambda x: x['width']) - self.entry = sorted(self.entry, key=lambda x: x['square']) + self.entry = sorted(self.entry, key=lambda x: x["square"]) self.entry.reverse() def sizes(self): """ Get a list of all available icon sizes and color depths. """ - return {(h['width'], h['height']) for h in self.entry} + return {(h["width"], h["height"]) for h in self.entry} + + def getentryindex(self, size, bpp=False): + for (i, h) in enumerate(self.entry): + if size == h["dim"] and (bpp is False or bpp == h["color_depth"]): + return i + return 0 def getimage(self, size, bpp=False): """ Get an image from the icon """ - for (i, h) in enumerate(self.entry): - if size == h['dim'] and (bpp is False or bpp == h['color_depth']): - return self.frame(i) - return self.frame(0) + return self.frame(self.getentryindex(size, bpp)) def frame(self, idx): """ @@ -157,9 +170,9 @@ class IcoFile(object): header = self.entry[idx] - self.buf.seek(header['offset']) + self.buf.seek(header["offset"]) data = self.buf.read(8) - self.buf.seek(header['offset']) + self.buf.seek(header["offset"]) if data[:8] == PngImagePlugin._MAGIC: # png frame @@ -167,6 +180,7 @@ class IcoFile(object): else: # XOR + AND mask bmp frame im = BmpImagePlugin.DibImageFile(self.buf) + Image._decompression_bomb_check(im.size) # change tile dimension to only encompass XOR image im._size = (im.size[0], int(im.size[1] / 2)) @@ -194,11 +208,11 @@ class IcoFile(object): # convert to an 8bpp grayscale image mask = Image.frombuffer( - 'L', # 8bpp - im.size, # (w, h) - alpha_bytes, # source chars - 'raw', # raw decoder - ('L', 0, -1) # 8bpp inverted, unpadded, reversed + "L", # 8bpp + im.size, # (w, h) + alpha_bytes, # source chars + "raw", # raw decoder + ("L", 0, -1), # 8bpp inverted, unpadded, reversed ) else: # get AND image from end of bitmap @@ -210,8 +224,7 @@ class IcoFile(object): # the total mask data is # padded row size * height / bits per char - and_mask_offset = o + int(im.size[0] * im.size[1] * - (bpp / 8.0)) + and_mask_offset = o + int(im.size[0] * im.size[1] * (bpp / 8.0)) total_bytes = int((w * im.size[1]) / 8) self.buf.seek(and_mask_offset) @@ -219,17 +232,17 @@ class IcoFile(object): # convert raw data to image mask = Image.frombuffer( - '1', # 1 bpp - im.size, # (w, h) - mask_data, # source chars - 'raw', # raw decoder - ('1;I', int(w/8), -1) # 1bpp inverted, padded, reversed + "1", # 1 bpp + im.size, # (w, h) + mask_data, # source chars + "raw", # raw decoder + ("1;I", int(w / 8), -1), # 1bpp inverted, padded, reversed ) # now we have two images, im is XOR image and mask is AND image # apply mask image as alpha channel - im = im.convert('RGBA') + im = im.convert("RGBA") im.putalpha(mask) return im @@ -238,6 +251,7 @@ class IcoFile(object): ## # Image plugin for Windows Icon files. + class IcoImageFile(ImageFile.ImageFile): """ PIL read-only image support for Microsoft Windows .ico files. @@ -250,17 +264,21 @@ class IcoImageFile(ImageFile.ImageFile): Handles classic, XP and Vista icon formats. + When saving, PNG compression is used. Support for this was only added in + Windows Vista. + This plugin is a refactored version of Win32IconImagePlugin by Bryan Davis . https://code.google.com/archive/p/casadebender/wikis/Win32IconImagePlugin.wiki """ + format = "ICO" format_description = "Windows Icon" def _open(self): self.ico = IcoFile(self.fp) - self.info['sizes'] = self.ico.sizes() - self.size = self.ico.entry[0]['dim'] + self.info["sizes"] = self.ico.sizes() + self.size = self.ico.entry[0]["dim"] self.load() @property @@ -269,23 +287,35 @@ class IcoImageFile(ImageFile.ImageFile): @size.setter def size(self, value): - if value not in self.info['sizes']: - raise ValueError( - "This is not one of the allowed sizes of this image") + if value not in self.info["sizes"]: + raise ValueError("This is not one of the allowed sizes of this image") self._size = value def load(self): + if self.im and self.im.size == self.size: + # Already loaded + return im = self.ico.getimage(self.size) # if tile is PNG, it won't really be loaded yet im.load() self.im = im.im self.mode = im.mode - self.size = im.size + if im.size != self.size: + warnings.warn("Image was not the expected size") + + index = self.ico.getentryindex(self.size) + sizes = list(self.info["sizes"]) + sizes[index] = im.size + self.info["sizes"] = set(sizes) + + self.size = im.size def load_seek(self): # Flag the ImageFile.Parser so that it # just does all the decode at the end. pass + + # # -------------------------------------------------------------------- @@ -293,3 +323,5 @@ class IcoImageFile(ImageFile.ImageFile): Image.register_open(IcoImageFile.format, IcoImageFile, _accept) Image.register_save(IcoImageFile.format, _save) Image.register_extension(IcoImageFile.format, ".ico") + +Image.register_mime(IcoImageFile.format, "image/x-icon") diff --git a/server/www/packages/packages-linux/x64/PIL/ImImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/ImImagePlugin.py index 2896bb4..77127fa 100644 --- a/server/www/packages/packages-linux/x64/PIL/ImImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/ImImagePlugin.py @@ -27,9 +27,12 @@ import re + from . import Image, ImageFile, ImagePalette from ._binary import i8 +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.7" @@ -46,8 +49,17 @@ SCALE = "Scale (x,y)" SIZE = "Image size (x*y)" MODE = "Image type" -TAGS = {COMMENT: 0, DATE: 0, EQUIPMENT: 0, FRAMES: 0, LUT: 0, NAME: 0, - SCALE: 0, SIZE: 0, MODE: 0} +TAGS = { + COMMENT: 0, + DATE: 0, + EQUIPMENT: 0, + FRAMES: 0, + LUT: 0, + NAME: 0, + SCALE: 0, + SIZE: 0, + MODE: 0, +} OPEN = { # ifunc93/p3cfunc formats @@ -69,6 +81,7 @@ OPEN = { "RYB3 image": ("RGB", "RYB;T"), # extensions "LA image": ("LA", "LA;L"), + "PA image": ("LA", "PA;L"), "RGBA image": ("RGBA", "RGBA;L"), "RGBX image": ("RGBX", "RGBX;L"), "CMYK image": ("CMYK", "CMYK;L"), @@ -105,6 +118,7 @@ def number(s): ## # Image plugin for the IFUNC IM file format. + class ImImageFile(ImageFile.ImageFile): format = "IM" @@ -137,7 +151,7 @@ class ImImageFile(ImageFile.ImageFile): if s == b"\r": continue - if not s or s == b'\0' or s == b'\x1A': + if not s or s == b"\0" or s == b"\x1A": break # FIXME: this may read whole file if not a text file @@ -146,14 +160,14 @@ class ImImageFile(ImageFile.ImageFile): if len(s) > 100: raise SyntaxError("not an IM file") - if s[-2:] == b'\r\n': + if s[-2:] == b"\r\n": s = s[:-2] - elif s[-1:] == b'\n': + elif s[-1:] == b"\n": s = s[:-1] try: m = split.match(s) - except re.error as v: + except re.error: raise SyntaxError("not an IM file") if m: @@ -162,8 +176,8 @@ class ImImageFile(ImageFile.ImageFile): # Don't know if this is the correct encoding, # but a decent guess (I guess) - k = k.decode('latin-1', 'replace') - v = v.decode('latin-1', 'replace') + k = k.decode("latin-1", "replace") + v = v.decode("latin-1", "replace") # Convert value as appropriate if k in [FRAMES, SCALE, SIZE]: @@ -189,8 +203,9 @@ class ImImageFile(ImageFile.ImageFile): else: - raise SyntaxError("Syntax error in IM header: " + - s.decode('ascii', 'replace')) + raise SyntaxError( + "Syntax error in IM header: " + s.decode("ascii", "replace") + ) if not n: raise SyntaxError("Not an IM file") @@ -200,7 +215,7 @@ class ImImageFile(ImageFile.ImageFile): self.mode = self.info[MODE] # Skip forward to start of image data - while s and s[0:1] != b'\x1A': + while s and s[0:1] != b"\x1A": s = self.fp.read(1) if not s: raise SyntaxError("File truncated") @@ -211,20 +226,21 @@ class ImImageFile(ImageFile.ImageFile): greyscale = 1 # greyscale palette linear = 1 # linear greyscale palette for i in range(256): - if palette[i] == palette[i+256] == palette[i+512]: + if palette[i] == palette[i + 256] == palette[i + 512]: if i8(palette[i]) != i: linear = 0 else: greyscale = 0 - if self.mode == "L" or self.mode == "LA": + if self.mode in ["L", "LA", "P", "PA"]: if greyscale: if not linear: self.lut = [i8(c) for c in palette[:256]] else: - if self.mode == "L": + if self.mode in ["L", "P"]: self.mode = self.rawmode = "P" - elif self.mode == "LA": - self.mode = self.rawmode = "PA" + elif self.mode in ["LA", "PA"]: + self.mode = "PA" + self.rawmode = "PA;L" self.palette = ImagePalette.raw("RGB;L", palette) elif self.mode == "RGB": if not greyscale or not linear: @@ -243,8 +259,7 @@ class ImImageFile(ImageFile.ImageFile): # use bit decoder (if necessary) bits = int(self.rawmode[2:]) if bits not in [8, 16, 32]: - self.tile = [("bit", (0, 0)+self.size, offs, - (bits, 8, 3, 0, -1))] + self.tile = [("bit", (0, 0) + self.size, offs, (bits, 8, 3, 0, -1))] return except ValueError: pass @@ -253,13 +268,14 @@ class ImImageFile(ImageFile.ImageFile): # Old LabEye/3PC files. Would be very surprised if anyone # ever stumbled upon such a file ;-) size = self.size[0] * self.size[1] - self.tile = [("raw", (0, 0)+self.size, offs, ("G", 0, -1)), - ("raw", (0, 0)+self.size, offs+size, ("R", 0, -1)), - ("raw", (0, 0)+self.size, offs+2*size, ("B", 0, -1))] + self.tile = [ + ("raw", (0, 0) + self.size, offs, ("G", 0, -1)), + ("raw", (0, 0) + self.size, offs + size, ("R", 0, -1)), + ("raw", (0, 0) + self.size, offs + 2 * size, ("B", 0, -1)), + ] else: # LabEye/IFUNC files - self.tile = [("raw", (0, 0)+self.size, offs, - (self.rawmode, 0, -1))] + self.tile = [("raw", (0, 0) + self.size, offs, (self.rawmode, 0, -1))] @property def n_frames(self): @@ -285,11 +301,21 @@ class ImImageFile(ImageFile.ImageFile): self.fp = self.__fp - self.tile = [("raw", (0, 0)+self.size, offs, (self.rawmode, 0, -1))] + self.tile = [("raw", (0, 0) + self.size, offs, (self.rawmode, 0, -1))] def tell(self): return self.frame + def _close__fp(self): + try: + if self.__fp != self.fp: + self.__fp.close() + except AttributeError: + pass + finally: + self.__fp = None + + # # -------------------------------------------------------------------- # Save IM files @@ -311,7 +337,7 @@ SAVE = { "RGBA": ("RGBA", "RGBA;L"), "RGBX": ("RGBX", "RGBX;L"), "CMYK": ("CMYK", "CMYK;L"), - "YCbCr": ("YCC", "YCbCr;L") + "YCbCr": ("YCC", "YCbCr;L"), } @@ -324,17 +350,18 @@ def _save(im, fp, filename): frames = im.encoderinfo.get("frames", 1) - fp.write(("Image type: %s image\r\n" % image_type).encode('ascii')) + fp.write(("Image type: %s image\r\n" % image_type).encode("ascii")) if filename: - fp.write(("Name: %s\r\n" % filename).encode('ascii')) - fp.write(("Image size (x*y): %d*%d\r\n" % im.size).encode('ascii')) - fp.write(("File size (no of images): %d\r\n" % frames).encode('ascii')) - if im.mode == "P": + fp.write(("Name: %s\r\n" % filename).encode("ascii")) + fp.write(("Image size (x*y): %d*%d\r\n" % im.size).encode("ascii")) + fp.write(("File size (no of images): %d\r\n" % frames).encode("ascii")) + if im.mode in ["P", "PA"]: fp.write(b"Lut: 1\r\n") - fp.write(b"\000" * (511-fp.tell()) + b"\032") - if im.mode == "P": + fp.write(b"\000" * (511 - fp.tell()) + b"\032") + if im.mode in ["P", "PA"]: fp.write(im.im.getpalette("RGB", "RGB;L")) # 768 bytes - ImageFile._save(im, fp, [("raw", (0, 0)+im.size, 0, (rawmode, 0, -1))]) + ImageFile._save(im, fp, [("raw", (0, 0) + im.size, 0, (rawmode, 0, -1))]) + # # -------------------------------------------------------------------- diff --git a/server/www/packages/packages-linux/x64/PIL/Image.py b/server/www/packages/packages-linux/x64/PIL/Image.py index edea312..0cdfcc9 100644 --- a/server/www/packages/packages-linux/x64/PIL/Image.py +++ b/server/www/packages/packages-linux/x64/PIL/Image.py @@ -24,15 +24,41 @@ # See the README file for information on usage and redistribution. # -# VERSION is deprecated and will be removed in Pillow 6.0.0. -# PILLOW_VERSION is deprecated and will be removed after that. -# Use __version__ instead. -from . import VERSION, PILLOW_VERSION, __version__, _plugins -from ._util import py3 - +import atexit +import io import logging -import warnings import math +import numbers +import os +import struct +import sys +import warnings + +# VERSION was removed in Pillow 6.0.0. +# PILLOW_VERSION is deprecated and will be removed in Pillow 7.0.0. +# Use __version__ instead. +from . import PILLOW_VERSION, ImageMode, TiffTags, __version__, _plugins +from ._binary import i8, i32le +from ._util import deferred_error, isPath, isStringType, py3 + +try: + import builtins +except ImportError: + import __builtin__ + + builtins = __builtin__ + + +try: + # Python 3 + from collections.abc import Callable, MutableMapping +except ImportError: + # Python 2.7 + from collections import Callable, MutableMapping + + +# Silence warning +assert PILLOW_VERSION logger = logging.getLogger(__name__) @@ -62,13 +88,13 @@ try: # Also note that Image.core is not a publicly documented interface, # and should be considered private and subject to change. from . import _imaging as core - if __version__ != getattr(core, 'PILLOW_VERSION', None): - raise ImportError("The _imaging extension was built for another " - "version of Pillow or PIL:\n" - "Core version: %s\n" - "Pillow version: %s" % - (getattr(core, 'PILLOW_VERSION', None), - __version__)) + + if __version__ != getattr(core, "PILLOW_VERSION", None): + raise ImportError( + "The _imaging extension was built for another version of Pillow or PIL:\n" + "Core version: %s\n" + "Pillow version: %s" % (getattr(core, "PILLOW_VERSION", None), __version__) + ) except ImportError as v: core = _imaging_not_installed() @@ -78,10 +104,9 @@ except ImportError as v: # the right version (windows only). Print a warning, if # possible. warnings.warn( - "The _imaging extension was built for another version " - "of Python.", - RuntimeWarning - ) + "The _imaging extension was built for another version of Python.", + RuntimeWarning, + ) elif str(v).startswith("The _imaging extension"): warnings.warn(str(v), RuntimeWarning) elif "Symbol not found: _PyUnicodeUCS2_" in str(v): @@ -90,60 +115,36 @@ except ImportError as v: warnings.warn( "The _imaging extension was built for Python with UCS2 support; " "recompile Pillow or build Python --without-wide-unicode. ", - RuntimeWarning - ) + RuntimeWarning, + ) elif "Symbol not found: _PyUnicodeUCS4_" in str(v): # should match _PyUnicodeUCS4_FromString and # _PyUnicodeUCS4_AsLatin1String warnings.warn( "The _imaging extension was built for Python with UCS4 support; " "recompile Pillow or build Python --with-wide-unicode. ", - RuntimeWarning - ) + RuntimeWarning, + ) # Fail here anyway. Don't let people run with a mostly broken Pillow. # see docs/porting.rst raise -try: - import builtins -except ImportError: - import __builtin__ - builtins = __builtin__ - -from . import ImageMode -from ._binary import i8 -from ._util import isPath, isStringType, deferred_error - -import os -import sys -import io -import struct -import atexit - -# type stuff -import numbers -try: - # Python 3 - from collections.abc import Callable -except ImportError: - # Python 2.7 - from collections import Callable - # works everywhere, win for pypy, not cpython -USE_CFFI_ACCESS = hasattr(sys, 'pypy_version_info') +USE_CFFI_ACCESS = hasattr(sys, "pypy_version_info") try: import cffi - HAS_CFFI = True except ImportError: - HAS_CFFI = False + cffi = None try: from pathlib import Path + HAS_PATHLIB = True except ImportError: try: from pathlib2 import Path + HAS_PATHLIB = True except ImportError: HAS_PATHLIB = False @@ -164,7 +165,7 @@ def isImageType(t): # -# Constants (also defined in _imagingmodule.c!) +# Constants NONE = 0 @@ -177,14 +178,14 @@ ROTATE_270 = 4 TRANSPOSE = 5 TRANSVERSE = 6 -# transforms +# transforms (also defined in Imaging.h) AFFINE = 0 EXTENT = 1 PERSPECTIVE = 2 QUAD = 3 MESH = 4 -# resampling filters +# resampling filters (also defined in Imaging.h) NEAREST = NONE = 0 BOX = 4 BILINEAR = LINEAR = 2 @@ -212,7 +213,7 @@ NORMAL = 0 SEQUENCE = 1 CONTAINER = 2 -if hasattr(core, 'DEFAULT_STRATEGY'): +if hasattr(core, "DEFAULT_STRATEGY"): DEFAULT_STRATEGY = core.DEFAULT_STRATEGY FILTERED = core.FILTERED HUFFMAN_ONLY = core.HUFFMAN_ONLY @@ -238,13 +239,12 @@ ENCODERS = {} _MODEINFO = { # NOTE: this table will be removed in future versions. use # getmode* functions or ImageMode descriptors instead. - # official modes "1": ("L", "L", ("1",)), "L": ("L", "L", ("L",)), "I": ("L", "I", ("I",)), "F": ("L", "F", ("F",)), - "P": ("RGB", "L", ("P",)), + "P": ("P", "L", ("P",)), "RGB": ("RGB", "L", ("R", "G", "B")), "RGBX": ("RGB", "L", ("R", "G", "B", "X")), "RGBA": ("RGB", "L", ("R", "G", "B", "A")), @@ -252,46 +252,44 @@ _MODEINFO = { "YCbCr": ("RGB", "L", ("Y", "Cb", "Cr")), "LAB": ("RGB", "L", ("L", "A", "B")), "HSV": ("RGB", "L", ("H", "S", "V")), - # Experimental modes include I;16, I;16L, I;16B, RGBa, BGR;15, and # BGR;24. Use these modes only if you know exactly what you're # doing... - } -if sys.byteorder == 'little': - _ENDIAN = '<' +if sys.byteorder == "little": + _ENDIAN = "<" else: - _ENDIAN = '>' + _ENDIAN = ">" _MODE_CONV = { # official modes - "1": ('|b1', None), # Bits need to be extended to bytes - "L": ('|u1', None), - "LA": ('|u1', 2), - "I": (_ENDIAN + 'i4', None), - "F": (_ENDIAN + 'f4', None), - "P": ('|u1', None), - "RGB": ('|u1', 3), - "RGBX": ('|u1', 4), - "RGBA": ('|u1', 4), - "CMYK": ('|u1', 4), - "YCbCr": ('|u1', 3), - "LAB": ('|u1', 3), # UNDONE - unsigned |u1i1i1 - "HSV": ('|u1', 3), + "1": ("|b1", None), # Bits need to be extended to bytes + "L": ("|u1", None), + "LA": ("|u1", 2), + "I": (_ENDIAN + "i4", None), + "F": (_ENDIAN + "f4", None), + "P": ("|u1", None), + "RGB": ("|u1", 3), + "RGBX": ("|u1", 4), + "RGBA": ("|u1", 4), + "CMYK": ("|u1", 4), + "YCbCr": ("|u1", 3), + "LAB": ("|u1", 3), # UNDONE - unsigned |u1i1i1 + "HSV": ("|u1", 3), # I;16 == I;16L, and I;32 == I;32L - "I;16": ('u2', None), - "I;16L": ('i2', None), - "I;16LS": ('u4', None), - "I;32L": ('i4', None), - "I;32LS": ('u2", None), + "I;16L": ("i2", None), + "I;16LS": ("u4", None), + "I;32L": ("i4", None), + "I;32LS": ("= 3: + def __del__(self): - if (hasattr(self, 'fp') and hasattr(self, '_exclusive_fp') - and self.fp and self._exclusive_fp): - self.fp.close() - self.fp = None + self.__exit__() def _copy(self): self.load() @@ -619,9 +644,9 @@ class Image(object): def _dump(self, file=None, format=None, **options): import tempfile - suffix = '' + suffix = "" if format: - suffix = '.'+format + suffix = "." + format if not file: f, filename = tempfile.mkstemp(suffix) @@ -641,35 +666,38 @@ class Image(object): return filename def __eq__(self, other): - return (isinstance(other, Image) and - self.__class__.__name__ == other.__class__.__name__ and - self.mode == other.mode and - self.size == other.size and - self.info == other.info and - self.category == other.category and - self.readonly == other.readonly and - self.getpalette() == other.getpalette() and - self.tobytes() == other.tobytes()) + return ( + self.__class__ is other.__class__ + and self.mode == other.mode + and self.size == other.size + and self.info == other.info + and self.category == other.category + and self.readonly == other.readonly + and self.getpalette() == other.getpalette() + and self.tobytes() == other.tobytes() + ) def __ne__(self, other): - eq = (self == other) + eq = self == other return not eq def __repr__(self): return "<%s.%s image mode=%s size=%dx%d at 0x%X>" % ( - self.__class__.__module__, self.__class__.__name__, - self.mode, self.size[0], self.size[1], - id(self) - ) + self.__class__.__module__, + self.__class__.__name__, + self.mode, + self.size[0], + self.size[1], + id(self), + ) def _repr_png_(self): """ iPython display hook support :returns: png version of the image as bytes """ - from io import BytesIO - b = BytesIO() - self.save(b, 'PNG') + b = io.BytesIO() + self.save(b, "PNG") return b.getvalue() @property @@ -677,24 +705,19 @@ class Image(object): # numpy array interface support new = {} shape, typestr = _conv_type_shape(self) - new['shape'] = shape - new['typestr'] = typestr - new['version'] = 3 - if self.mode == '1': + new["shape"] = shape + new["typestr"] = typestr + new["version"] = 3 + if self.mode == "1": # Binary images need to be extended from bits to bytes # See: https://github.com/python-pillow/Pillow/issues/350 - new['data'] = self.tobytes('raw', 'L') + new["data"] = self.tobytes("raw", "L") else: - new['data'] = self.tobytes() + new["data"] = self.tobytes() return new def __getstate__(self): - return [ - self.info, - self.mode, - self.size, - self.getpalette(), - self.tobytes()] + return [self.info, self.mode, self.size, self.getpalette(), self.tobytes()] def __setstate__(self, state): Image.__init__(self) @@ -704,7 +727,7 @@ class Image(object): self.mode = mode self._size = size self.im = core.new(mode, size) - if mode in ("L", "P") and palette: + if mode in ("L", "LA", "P", "PA") and palette: self.putpalette(palette) self.frombytes(data) @@ -752,8 +775,9 @@ class Image(object): return b"".join(data) def tostring(self, *args, **kw): - raise NotImplementedError("tostring() has been removed. " - "Please call tobytes() instead.") + raise NotImplementedError( + "tostring() has been removed. Please call tobytes() instead." + ) def tobitmap(self, name="image"): """ @@ -770,11 +794,15 @@ class Image(object): if self.mode != "1": raise ValueError("not a bitmap") data = self.tobytes("xbm") - return b"".join([ - ("#define %s_width %d\n" % (name, self.size[0])).encode('ascii'), - ("#define %s_height %d\n" % (name, self.size[1])).encode('ascii'), - ("static char %s_bits[] = {\n" % name).encode('ascii'), data, b"};" - ]) + return b"".join( + [ + ("#define %s_width %d\n" % (name, self.size[0])).encode("ascii"), + ("#define %s_height %d\n" % (name, self.size[1])).encode("ascii"), + ("static char %s_bits[] = {\n" % name).encode("ascii"), + data, + b"};", + ] + ) def frombytes(self, data, decoder_name="raw", *args): """ @@ -803,8 +831,9 @@ class Image(object): raise ValueError("cannot decode image data") def fromstring(self, *args, **kw): - raise NotImplementedError("fromstring() has been removed. " - "Please call frombytes() instead.") + raise NotImplementedError( + "fromstring() has been removed. Please call frombytes() instead." + ) def load(self): """ @@ -813,8 +842,10 @@ class Image(object): Image class automatically loads an opened image when it is accessed for the first time. - This method will close the file associated with the image. See - :ref:`file-handling` for more information. + If the file associated with the image was opened by Pillow, then this + method will close it. The exception to this is if the image has + multiple frames, in which case the file will be left open for seek + operations. See :ref:`file-handling` for more information. :returns: An image access object. :rtype: :ref:`PixelAccess` or :py:class:`PIL.PyAccess` @@ -833,10 +864,11 @@ class Image(object): self.palette.mode = "RGBA" if self.im: - if HAS_CFFI and USE_CFFI_ACCESS: + if cffi and USE_CFFI_ACCESS: if self.pyaccess: return self.pyaccess from . import PyAccess + self.pyaccess = PyAccess.new(self, self.readonly) if self.pyaccess: return self.pyaccess @@ -853,8 +885,7 @@ class Image(object): """ pass - def convert(self, mode=None, matrix=None, dither=None, - palette=WEB, colors=256): + def convert(self, mode=None, matrix=None, dither=None, palette=WEB, colors=256): """ Returns a converted copy of this image. For the "P" mode, this method translates pixels through the palette. If mode is @@ -865,7 +896,7 @@ class Image(object): "L", "RGB" and "CMYK." The **matrix** argument only supports "L" and "RGB". - When translating a color image to black and white (mode "L"), + When translating a color image to greyscale (mode "L"), the library uses the ITU-R 601-2 luma transform:: L = R * 299/1000 + G * 587/1000 + B * 114/1000 @@ -873,9 +904,9 @@ class Image(object): The default method of converting a greyscale ("L") or "RGB" image into a bilevel (mode "1") image uses Floyd-Steinberg dither to approximate the original image luminosity levels. If - dither is NONE, all non-zero values are set to 255 (white). To - use other thresholds, use the :py:meth:`~PIL.Image.Image.point` - method. + dither is NONE, all values larger than 128 are set to 255 (white), + all other values to 0 (black). To use other thresholds, use the + :py:meth:`~PIL.Image.Image.point` method. When converting from "RGBA" to "P" without a **matrix** argument, this passes the operation to :py:meth:`~PIL.Image.Image.quantize`, @@ -907,7 +938,7 @@ class Image(object): if not mode or (mode == self.mode and not matrix): return self.copy() - has_transparency = self.info.get('transparency') is not None + has_transparency = self.info.get("transparency") is not None if matrix: # matrix conversion if mode not in ("L", "RGB"): @@ -915,19 +946,24 @@ class Image(object): im = self.im.convert_matrix(mode, matrix) new = self._new(im) if has_transparency and self.im.bands == 3: - transparency = new.info['transparency'] + transparency = new.info["transparency"] def convert_transparency(m, v): - v = m[0]*v[0] + m[1]*v[1] + m[2]*v[2] + m[3]*0.5 + v = m[0] * v[0] + m[1] * v[1] + m[2] * v[2] + m[3] * 0.5 return max(0, min(255, int(v))) + if mode == "L": transparency = convert_transparency(matrix, transparency) elif len(mode) == 3: - transparency = tuple([ - convert_transparency(matrix[i*4:i*4+4], transparency) - for i in range(0, len(transparency)) - ]) - new.info['transparency'] = transparency + transparency = tuple( + [ + convert_transparency( + matrix[i * 4 : i * 4 + 4], transparency + ) + for i in range(0, len(transparency)) + ] + ) + new.info["transparency"] = transparency return new if mode == "P" and self.mode == "RGBA": @@ -937,45 +973,48 @@ class Image(object): delete_trns = False # transparency handling if has_transparency: - if self.mode in ('L', 'RGB') and mode == 'RGBA': + if self.mode in ("1", "L", "I", "RGB") and mode == "RGBA": # Use transparent conversion to promote from transparent # color to an alpha channel. - new_im = self._new(self.im.convert_transparent( - mode, self.info['transparency'])) - del(new_im.info['transparency']) + new_im = self._new( + self.im.convert_transparent(mode, self.info["transparency"]) + ) + del new_im.info["transparency"] return new_im - elif self.mode in ('L', 'RGB', 'P') and mode in ('L', 'RGB', 'P'): - t = self.info['transparency'] + elif self.mode in ("L", "RGB", "P") and mode in ("L", "RGB", "P"): + t = self.info["transparency"] if isinstance(t, bytes): # Dragons. This can't be represented by a single color - warnings.warn('Palette images with Transparency ' + - ' expressed in bytes should be converted ' + - 'to RGBA images') + warnings.warn( + "Palette images with Transparency expressed in bytes should be " + "converted to RGBA images" + ) delete_trns = True else: # get the new transparency color. # use existing conversions trns_im = Image()._new(core.new(self.mode, (1, 1))) - if self.mode == 'P': + if self.mode == "P": trns_im.putpalette(self.palette) if isinstance(t, tuple): try: t = trns_im.palette.getcolor(t) - except: - raise ValueError("Couldn't allocate a palette " - "color for transparency") + except Exception: + raise ValueError( + "Couldn't allocate a palette color for transparency" + ) trns_im.putpixel((0, 0), t) - if mode in ('L', 'RGB'): + if mode in ("L", "RGB"): trns_im = trns_im.convert(mode) else: # can't just retrieve the palette number, got to do it # after quantization. - trns_im = trns_im.convert('RGB') + trns_im = trns_im.convert("RGB") trns = trns_im.getpixel((0, 0)) - elif self.mode == 'P' and mode == 'RGBA': - t = self.info['transparency'] + elif self.mode == "P" and mode == "RGBA": + t = self.info["transparency"] delete_trns = True if isinstance(t, bytes): @@ -983,27 +1022,26 @@ class Image(object): elif isinstance(t, int): self.im.putpalettealpha(t, 0) else: - raise ValueError("Transparency for P mode should" + - " be bytes or int") + raise ValueError("Transparency for P mode should be bytes or int") if mode == "P" and palette == ADAPTIVE: im = self.im.quantize(colors) new = self._new(im) from . import ImagePalette + new.palette = ImagePalette.raw("RGB", new.im.getpalette("RGB")) if delete_trns: # This could possibly happen if we requantize to fewer colors. # The transparency would be totally off in that case. - del(new.info['transparency']) + del new.info["transparency"] if trns is not None: try: - new.info['transparency'] = new.palette.getcolor(trns) - except: + new.info["transparency"] = new.palette.getcolor(trns) + except Exception: # if we can't make a transparent color, don't leave the old # transparency hanging around to mess us up. - del(new.info['transparency']) - warnings.warn("Couldn't allocate palette entry " + - "for transparency") + del new.info["transparency"] + warnings.warn("Couldn't allocate palette entry for transparency") return new # colorspace conversion @@ -1023,20 +1061,19 @@ class Image(object): new_im = self._new(im) if delete_trns: # crash fail if we leave a bytes transparency in an rgb/l mode. - del(new_im.info['transparency']) + del new_im.info["transparency"] if trns is not None: - if new_im.mode == 'P': + if new_im.mode == "P": try: - new_im.info['transparency'] = new_im.palette.getcolor(trns) - except: - del(new_im.info['transparency']) - warnings.warn("Couldn't allocate palette entry " + - "for transparency") + new_im.info["transparency"] = new_im.palette.getcolor(trns) + except Exception: + del new_im.info["transparency"] + warnings.warn("Couldn't allocate palette entry for transparency") else: - new_im.info['transparency'] = trns + new_im.info["transparency"] = trns return new_im - def quantize(self, colors=256, method=None, kmeans=0, palette=None): + def quantize(self, colors=256, method=None, kmeans=0, palette=None, dither=1): """ Convert the image to 'P' mode with the specified number of colors. @@ -1047,7 +1084,12 @@ class Image(object): 2 = fast octree 3 = libimagequant :param kmeans: Integer - :param palette: Quantize to the palette of given :py:class:`PIL.Image.Image`. + :param palette: Quantize to the palette of given + :py:class:`PIL.Image.Image`. + :param dither: Dithering method, used when converting from + mode "RGB" to "P" or from "RGB" or "L" to "1". + Available methods are NONE or FLOYDSTEINBERG (default). + Default: 1 (legacy setting) :returns: A new image """ @@ -1057,14 +1099,15 @@ class Image(object): if method is None: # defaults: method = 0 - if self.mode == 'RGBA': + if self.mode == "RGBA": method = 2 - if self.mode == 'RGBA' and method not in (2, 3): + if self.mode == "RGBA" and method not in (2, 3): # Caller specified an invalid mode. raise ValueError( - 'Fast Octree (method == 2) and libimagequant (method == 3) ' + - 'are the only valid methods for quantizing RGBA images') + "Fast Octree (method == 2) and libimagequant (method == 3) " + "are the only valid methods for quantizing RGBA images" + ) if palette: # use palette from reference image @@ -1074,11 +1117,18 @@ class Image(object): if self.mode != "RGB" and self.mode != "L": raise ValueError( "only RGB or L mode images can be quantized to a palette" - ) - im = self.im.convert("P", 1, palette.im) + ) + im = self.im.convert("P", dither, palette.im) return self._new(im) - return self._new(self.im.quantize(colors, method, kmeans)) + im = self._new(self.im.quantize(colors, method, kmeans)) + + from . import ImagePalette + + mode = im.im.getpalettemode() + im.palette = ImagePalette.ImagePalette(mode, im.im.getpalette(mode, mode)) + + return im def copy(self): """ @@ -1173,8 +1223,9 @@ class Image(object): if isinstance(filter, Callable): filter = filter() if not hasattr(filter, "filter"): - raise TypeError("filter argument should be ImageFilter.Filter " + - "instance or class") + raise TypeError( + "filter argument should be ImageFilter.Filter instance or class" + ) multiband = isinstance(filter, ImageFilter.MultibandFilter) if self.im.bands == 1 or multiband: @@ -1273,6 +1324,12 @@ class Image(object): return tuple(extrema) return self.im.getextrema() + def getexif(self): + if self._exif is None: + self._exif = Exif() + self._exif.load(self.info.get("exif")) + return self._exif + def getim(self): """ Returns a capsule that points to the internal image memory. @@ -1344,6 +1401,7 @@ class Image(object): bi-level image (mode "1") or a greyscale image ("L"). :param mask: An optional mask. + :param extrema: An optional tuple of manually-specified extrema. :returns: A list containing pixel counts. """ self.load() @@ -1356,9 +1414,36 @@ class Image(object): return self.im.histogram(extrema) return self.im.histogram() + def entropy(self, mask=None, extrema=None): + """ + Calculates and returns the entropy for the image. + + A bilevel image (mode "1") is treated as a greyscale ("L") + image by this method. + + If a mask is provided, the method employs the histogram for + those parts of the image where the mask image is non-zero. + The mask image must have the same size as the image, and be + either a bi-level image (mode "1") or a greyscale image ("L"). + + :param mask: An optional mask. + :param extrema: An optional tuple of manually-specified extrema. + :returns: A float value representing the image entropy + """ + self.load() + if mask: + mask.load() + return self.im.entropy((0, 0), mask.im) + if self.mode in ("I", "F"): + if extrema is None: + extrema = self.getextrema() + return self.im.entropy(extrema) + return self.im.entropy() + def offset(self, xoffset, yoffset=None): - raise NotImplementedError("offset() has been removed. " - "Please call ImageChops.offset() instead.") + raise NotImplementedError( + "offset() has been removed. Please call ImageChops.offset() instead." + ) def paste(self, im, box=None, mask=None): """ @@ -1416,13 +1501,12 @@ class Image(object): size = mask.size else: # FIXME: use self.size here? - raise ValueError( - "cannot determine region size; use 4-item box" - ) - box += (box[0]+size[0], box[1]+size[1]) + raise ValueError("cannot determine region size; use 4-item box") + box += (box[0] + size[0], box[1] + size[1]) if isStringType(im): from . import ImageColor + im = ImageColor.getcolor(im, self.mode) elif isImageType(im): @@ -1541,7 +1625,7 @@ class Image(object): self._ensure_mutable() - if self.mode not in ("LA", "RGBA"): + if self.mode not in ("LA", "PA", "RGBA"): # attempt to promote self to a matching alpha mode try: mode = getmodebase(self.mode) + "A" @@ -1550,7 +1634,7 @@ class Image(object): except (AttributeError, ValueError): # do things the hard way im = self.im.convert(mode) - if im.mode not in ("LA", "RGBA"): + if im.mode not in ("LA", "PA", "RGBA"): raise ValueError # sanity check self.im = im self.pyaccess = None @@ -1558,7 +1642,7 @@ class Image(object): except (KeyError, ValueError): raise ValueError("illegal image mode") - if self.mode == "LA": + if self.mode in ("LA", "PA"): band = 1 else: band = 3 @@ -1601,10 +1685,10 @@ class Image(object): def putpalette(self, data, rawmode="RGB"): """ - Attaches a palette to this image. The image must be a "P" or - "L" image, and the palette sequence must contain 768 integer - values, where each group of three values represent the red, - green, and blue values for the corresponding pixel + Attaches a palette to this image. The image must be a "P", + "PA", "L" or "LA" image, and the palette sequence must contain + 768 integer values, where each group of three values represent + the red, green, and blue values for the corresponding pixel index. Instead of an integer sequence, you can use an 8-bit string. @@ -1613,7 +1697,7 @@ class Image(object): """ from . import ImagePalette - if self.mode not in ("L", "P"): + if self.mode not in ("L", "LA", "P", "PA"): raise ValueError("illegal image mode") self.load() if isinstance(data, ImagePalette.ImagePalette): @@ -1625,7 +1709,7 @@ class Image(object): else: data = "".join(chr(x) for x in data) palette = ImagePalette.raw(rawmode, data) - self.mode = "P" + self.mode = "PA" if "A" in self.mode else "P" self.palette = palette self.palette.mode = "RGB" self.load() # install new palette @@ -1634,7 +1718,8 @@ class Image(object): """ Modifies the pixel at the given position. The color is given as a single numerical value for single-band images, and a tuple for - multi-band images. + multi-band images. In addition to this, RGB and RGBA tuples are + accepted for P images. Note that this method is relatively slow. For more extensive changes, use :py:meth:`~PIL.Image.Image.paste` or the :py:mod:`~PIL.ImageDraw` @@ -1657,6 +1742,14 @@ class Image(object): if self.pyaccess: return self.pyaccess.putpixel(xy, value) + + if ( + self.mode == "P" + and isinstance(value, (list, tuple)) + and len(value) in [3, 4] + ): + # RGB or RGBA value for a P image + value = self.palette.getcolor(value) return self.im.putpixel(xy, value) def remap_palette(self, dest_map, source_palette=None): @@ -1664,7 +1757,7 @@ class Image(object): Rewrites the image to reorder the palette. :param dest_map: A list of indexes into the original palette. - e.g. [1,0] would swap a two item palette, and list(range(255)) + e.g. [1,0] would swap a two item palette, and list(range(256)) is the identity transform. :param source_palette: Bytes or None. :returns: An :py:class:`~PIL.Image.Image` object. @@ -1679,16 +1772,16 @@ class Image(object): if self.mode == "P": real_source_palette = self.im.getpalette("RGB")[:768] else: # L-mode - real_source_palette = bytearray(i//3 for i in range(768)) + real_source_palette = bytearray(i // 3 for i in range(768)) else: real_source_palette = source_palette palette_bytes = b"" - new_positions = [0]*256 + new_positions = [0] * 256 # pick only the used colors from the palette for i, oldPosition in enumerate(dest_map): - palette_bytes += real_source_palette[oldPosition*3:oldPosition*3+3] + palette_bytes += real_source_palette[oldPosition * 3 : oldPosition * 3 + 3] new_positions[oldPosition] = i # replace the palette color id of all pixel with the new id @@ -1712,26 +1805,25 @@ class Image(object): mapping_palette = bytearray(new_positions) m_im = self.copy() - m_im.mode = 'P' + m_im.mode = "P" - m_im.palette = ImagePalette.ImagePalette("RGB", - palette=mapping_palette*3, - size=768) + m_im.palette = ImagePalette.ImagePalette( + "RGB", palette=mapping_palette * 3, size=768 + ) # possibly set palette dirty, then # m_im.putpalette(mapping_palette, 'L') # converts to 'P' # or just force it. # UNDONE -- this is part of the general issue with palettes m_im.im.putpalette(*m_im.palette.getdata()) - m_im = m_im.convert('L') + m_im = m_im.convert("L") # Internally, we require 768 bytes for a palette. - new_palette_bytes = (palette_bytes + - (768 - len(palette_bytes)) * b'\x00') + new_palette_bytes = palette_bytes + (768 - len(palette_bytes)) * b"\x00" m_im.putpalette(new_palette_bytes) - m_im.palette = ImagePalette.ImagePalette("RGB", - palette=palette_bytes, - size=len(palette_bytes)) + m_im.palette = ImagePalette.ImagePalette( + "RGB", palette=palette_bytes, size=len(palette_bytes) + ) return m_im @@ -1755,10 +1847,23 @@ class Image(object): :returns: An :py:class:`~PIL.Image.Image` object. """ - if resample not in ( - NEAREST, BILINEAR, BICUBIC, LANCZOS, BOX, HAMMING, - ): - raise ValueError("unknown resampling filter") + if resample not in (NEAREST, BILINEAR, BICUBIC, LANCZOS, BOX, HAMMING): + message = "Unknown resampling filter ({}).".format(resample) + + filters = [ + "{} ({})".format(filter[1], filter[0]) + for filter in ( + (NEAREST, "Image.NEAREST"), + (LANCZOS, "Image.LANCZOS"), + (BILINEAR, "Image.BILINEAR"), + (BICUBIC, "Image.BICUBIC"), + (BOX, "Image.BOX"), + (HAMMING, "Image.HAMMING"), + ) + ] + raise ValueError( + message + " Use " + ", ".join(filters[:-1]) + " or " + filters[-1] + ) size = tuple(size) @@ -1773,18 +1878,24 @@ class Image(object): if self.mode in ("1", "P"): resample = NEAREST - if self.mode == 'LA': - return self.convert('La').resize(size, resample, box).convert('LA') - - if self.mode == 'RGBA': - return self.convert('RGBa').resize(size, resample, box).convert('RGBA') + if self.mode in ["LA", "RGBA"]: + im = self.convert(self.mode[:-1] + "a") + im = im.resize(size, resample, box) + return im.convert(self.mode) self.load() return self._new(self.im.resize(size, resample, box)) - def rotate(self, angle, resample=NEAREST, expand=0, center=None, - translate=None, fillcolor=None): + def rotate( + self, + angle, + resample=NEAREST, + expand=0, + center=None, + translate=None, + fillcolor=None, + ): """ Returns a rotated copy of this image. This method returns a copy of this image, rotated the given number of degrees counter @@ -1849,22 +1960,28 @@ class Image(object): else: post_trans = translate if center is None: - rotn_center = (w / 2.0, h / 2.0) # FIXME These should be rounded to ints? + # FIXME These should be rounded to ints? + rotn_center = (w / 2.0, h / 2.0) else: rotn_center = center - angle = - math.radians(angle) + angle = -math.radians(angle) matrix = [ - round(math.cos(angle), 15), round(math.sin(angle), 15), 0.0, - round(-math.sin(angle), 15), round(math.cos(angle), 15), 0.0 + round(math.cos(angle), 15), + round(math.sin(angle), 15), + 0.0, + round(-math.sin(angle), 15), + round(math.cos(angle), 15), + 0.0, ] def transform(x, y, matrix): (a, b, c, d, e, f) = matrix - return a*x + b*y + c, d*x + e*y + f + return a * x + b * y + c, d * x + e * y + f - matrix[2], matrix[5] = transform(-rotn_center[0] - post_trans[0], - -rotn_center[1] - post_trans[1], matrix) + matrix[2], matrix[5] = transform( + -rotn_center[0] - post_trans[0], -rotn_center[1] - post_trans[1], matrix + ) matrix[2] += rotn_center[0] matrix[5] += rotn_center[1] @@ -1882,9 +1999,7 @@ class Image(object): # We multiply a translation matrix from the right. Because of its # special form, this is the same as taking the image of the # translation vector as new translation vector. - matrix[2], matrix[5] = transform(-(nw - w) / 2.0, - -(nh - h) / 2.0, - matrix) + matrix[2], matrix[5] = transform(-(nw - w) / 2.0, -(nh - h) / 2.0, matrix) w, h = nw, nh return self.transform((w, h), AFFINE, matrix, resample, fillcolor=fillcolor) @@ -1932,9 +2047,9 @@ class Image(object): filename = fp.name # may mutate self! - self.load() + self._ensure_mutable() - save_all = params.pop('save_all', False) + save_all = params.pop("save_all", False) self.encoderinfo = params self.encoderconfig = () @@ -1948,7 +2063,7 @@ class Image(object): try: format = EXTENSION[ext] except KeyError: - raise ValueError('unknown file extension: {}'.format(ext)) + raise ValueError("unknown file extension: {}".format(ext)) if format.upper() not in SAVE: init() @@ -1958,11 +2073,11 @@ class Image(object): save_handler = SAVE[format.upper()] if open_fp: - if params.get('append', False): - fp = builtins.open(filename, "r+b") - else: + if params.get("append", False): # Open also for reading ("+"), because TIFF save_all # writer needs to go back and edit the written data. + fp = builtins.open(filename, "r+b") + else: fp = builtins.open(filename, "w+b") try: @@ -1979,9 +2094,6 @@ class Image(object): **EOFError** exception. When a sequence file is opened, the library automatically seeks to frame 0. - Note that in the current version of the library, most sequence - formats only allows you to seek to the next frame. - See :py:meth:`~PIL.Image.Image.tell`. :param frame: Frame number, starting at 0. @@ -1998,15 +2110,15 @@ class Image(object): Displays this image. This method is mainly intended for debugging purposes. - On Unix platforms, this method saves the image to a temporary - PPM file, and calls either the **xv** utility or the **display** - utility, depending on which one can be found. + The image is first saved to a temporary file. By default, it will be in + PNG format. - On macOS, this method saves the image to a temporary BMP file, and - opens it with the native Preview application. + On Unix, the image is then opened using the **display**, **eog** or + **xv** utility, depending on which one can be found. - On Windows, it saves the image to a temporary BMP file, and uses - the standard BMP display utility to show it (usually Paint). + On macOS, the image is opened with the native Preview application. + + On Windows, the image is opened with the standard PNG display utility. :param title: Optional title to use for the image window, where possible. @@ -2053,8 +2165,7 @@ class Image(object): try: channel = self.getbands().index(channel) except ValueError: - raise ValueError( - 'The image has no channel "{}"'.format(channel)) + raise ValueError('The image has no channel "{}"'.format(channel)) return self._new(self.im.getband(channel)) @@ -2104,19 +2215,21 @@ class Image(object): self.draft(None, size) - im = self.resize(size, resample) + if self.size != size: + im = self.resize(size, resample) - self.im = im.im - self.mode = im.mode - self._size = size + self.im = im.im + self._size = size + self.mode = self.im.mode self.readonly = 0 self.pyaccess = None # FIXME: the different transform methods need further explanation # instead of bloating the method docs, add a separate chapter. - def transform(self, size, method, data=None, resample=NEAREST, - fill=1, fillcolor=None): + def transform( + self, size, method, data=None, resample=NEAREST, fill=1, fillcolor=None + ): """ Transforms this image. This method creates a new image with the given size, and the same mode as the original, and copies data @@ -2154,18 +2267,24 @@ class Image(object): :param fill: If **method** is an :py:class:`~PIL.Image.ImageTransformHandler` object, this is one of the arguments passed to it. Otherwise, it is unused. - :param fillcolor: Optional fill color for the area outside the transform - in the output image. + :param fillcolor: Optional fill color for the area outside the + transform in the output image. :returns: An :py:class:`~PIL.Image.Image` object. """ - if self.mode == 'LA': - return self.convert('La').transform( - size, method, data, resample, fill, fillcolor).convert('LA') + if self.mode == "LA": + return ( + self.convert("La") + .transform(size, method, data, resample, fill, fillcolor) + .convert("LA") + ) - if self.mode == 'RGBA': - return self.convert('RGBa').transform( - size, method, data, resample, fill, fillcolor).convert('RGBA') + if self.mode == "RGBA": + return ( + self.convert("RGBa") + .transform(size, method, data, resample, fill, fillcolor) + .convert("RGBA") + ) if isinstance(method, ImageTransformHandler): return method.transform(size, self, resample=resample, fill=fill) @@ -2181,16 +2300,15 @@ class Image(object): if method == MESH: # list of quads for box, quad in data: - im.__transformer(box, self, QUAD, quad, resample, - fillcolor is None) + im.__transformer(box, self, QUAD, quad, resample, fillcolor is None) else: - im.__transformer((0, 0)+size, self, method, data, - resample, fillcolor is None) + im.__transformer( + (0, 0) + size, self, method, data, resample, fillcolor is None + ) return im - def __transformer(self, box, image, method, data, - resample=NEAREST, fill=1): + def __transformer(self, box, image, method, data, resample=NEAREST, fill=1): w = box[2] - box[0] h = box[3] - box[1] @@ -2218,16 +2336,41 @@ class Image(object): x0, y0 = nw As = 1.0 / w At = 1.0 / h - data = (x0, (ne[0]-x0)*As, (sw[0]-x0)*At, - (se[0]-sw[0]-ne[0]+x0)*As*At, - y0, (ne[1]-y0)*As, (sw[1]-y0)*At, - (se[1]-sw[1]-ne[1]+y0)*As*At) + data = ( + x0, + (ne[0] - x0) * As, + (sw[0] - x0) * At, + (se[0] - sw[0] - ne[0] + x0) * As * At, + y0, + (ne[1] - y0) * As, + (sw[1] - y0) * At, + (se[1] - sw[1] - ne[1] + y0) * As * At, + ) else: raise ValueError("unknown transformation method") if resample not in (NEAREST, BILINEAR, BICUBIC): - raise ValueError("unknown resampling filter") + if resample in (BOX, HAMMING, LANCZOS): + message = { + BOX: "Image.BOX", + HAMMING: "Image.HAMMING", + LANCZOS: "Image.LANCZOS/Image.ANTIALIAS", + }[resample] + " ({}) cannot be used.".format(resample) + else: + message = "Unknown resampling filter ({}).".format(resample) + + filters = [ + "{} ({})".format(filter[1], filter[0]) + for filter in ( + (NEAREST, "Image.NEAREST"), + (BILINEAR, "Image.BILINEAR"), + (BICUBIC, "Image.BICUBIC"), + ) + ] + raise ValueError( + message + " Use " + ", ".join(filters[:-1]) + " or " + filters[-1] + ) image.load() @@ -2264,6 +2407,7 @@ class Image(object): def toqimage(self): """Returns a QImage copy of this image""" from . import ImageQt + if not ImageQt.qt_is_installed: raise ImportError("Qt bindings are not installed") return ImageQt.toqimage(self) @@ -2271,6 +2415,7 @@ class Image(object): def toqpixmap(self): """Returns a QPixmap copy of this image""" from . import ImageQt + if not ImageQt.qt_is_installed: raise ImportError("Qt bindings are not installed") return ImageQt.toqpixmap(self) @@ -2279,6 +2424,7 @@ class Image(object): # -------------------------------------------------------------------- # Abstract handlers. + class ImagePointHandler(object): # used as a mixin by point transforms (for use with im.point) pass @@ -2295,6 +2441,7 @@ class ImageTransformHandler(object): # # Debugging + def _wedge(): """Create greyscale wedge (for debugging only)""" @@ -2345,9 +2492,17 @@ def new(mode, size, color=0): # css3-style specifier from . import ImageColor + color = ImageColor.getcolor(color, mode) - return Image()._new(core.fill(mode, size, color)) + im = Image() + if mode == "P" and isinstance(color, (list, tuple)) and len(color) in [3, 4]: + # RGB or RGBA value for a P image + from . import ImagePalette + + im.palette = ImagePalette.ImagePalette() + color = im.palette.getcolor(color) + return im._new(core.fill(mode, size, color)) def frombytes(mode, size, data, decoder_name="raw", *args): @@ -2389,8 +2544,9 @@ def frombytes(mode, size, data, decoder_name="raw", *args): def fromstring(*args, **kw): - raise NotImplementedError("fromstring() has been removed. " + - "Please call frombytes() instead.") + raise NotImplementedError( + "fromstring() has been removed. Please call frombytes() instead." + ) def frombuffer(mode, size, data, decoder_name="raw", *args): @@ -2437,17 +2593,16 @@ def frombuffer(mode, size, data, decoder_name="raw", *args): if decoder_name == "raw": if args == (): warnings.warn( - "the frombuffer defaults may change in a future release; " + "the frombuffer defaults will change in Pillow 7.0.0; " "for portability, change the call to read:\n" " frombuffer(mode, size, data, 'raw', mode, 0, 1)", - RuntimeWarning, stacklevel=2 + RuntimeWarning, + stacklevel=2, ) args = mode, 0, -1 # may change to (mode, 0, 1) post-1.1.6 if args[0] in _MAPMODES: im = new(mode, (1, 1)) - im = im._new( - core.map_buffer(data, size, decoder_name, None, 0, args) - ) + im = im._new(core.map_buffer(data, size, decoder_name, None, 0, args)) im.readonly = 1 return im @@ -2481,12 +2636,12 @@ def fromarray(obj, mode=None): .. versionadded:: 1.1.6 """ arr = obj.__array_interface__ - shape = arr['shape'] + shape = arr["shape"] ndim = len(shape) - strides = arr.get('strides', None) + strides = arr.get("strides", None) if mode is None: try: - typekey = (1, 1) + shape[2:], arr['typestr'] + typekey = (1, 1) + shape[2:], arr["typestr"] mode, rawmode = _fromarray_typemap[typekey] except KeyError: raise TypeError("Cannot handle this data type") @@ -2503,7 +2658,7 @@ def fromarray(obj, mode=None): size = shape[1], shape[0] if strides is not None: - if hasattr(obj, 'tobytes'): + if hasattr(obj, "tobytes"): obj = obj.tobytes() else: obj = obj.tostring() @@ -2514,6 +2669,7 @@ def fromarray(obj, mode=None): def fromqimage(im): """Creates an image instance from a QImage image""" from . import ImageQt + if not ImageQt.qt_is_installed: raise ImportError("Qt bindings are not installed") return ImageQt.fromqimage(im) @@ -2522,6 +2678,7 @@ def fromqimage(im): def fromqpixmap(im): """Creates an image instance from a QPixmap image""" from . import ImageQt + if not ImageQt.qt_is_installed: raise ImportError("Qt bindings are not installed") return ImageQt.fromqpixmap(im) @@ -2548,7 +2705,7 @@ _fromarray_typemap = { ((1, 1, 2), "|u1"): ("LA", "LA"), ((1, 1, 3), "|u1"): ("RGB", "RGB"), ((1, 1, 4), "|u1"): ("RGBA", "RGBA"), - } +} # shortcuts _fromarray_typemap[((1, 1), _ENDIAN + "i4")] = ("I", "I") @@ -2564,15 +2721,15 @@ def _decompression_bomb_check(size): if pixels > 2 * MAX_IMAGE_PIXELS: raise DecompressionBombError( "Image size (%d pixels) exceeds limit of %d pixels, " - "could be decompression bomb DOS attack." % - (pixels, 2 * MAX_IMAGE_PIXELS)) + "could be decompression bomb DOS attack." % (pixels, 2 * MAX_IMAGE_PIXELS) + ) if pixels > MAX_IMAGE_PIXELS: warnings.warn( "Image size (%d pixels) exceeds limit of %d pixels, " - "could be decompression bomb DOS attack." % - (pixels, MAX_IMAGE_PIXELS), - DecompressionBombWarning) + "could be decompression bomb DOS attack." % (pixels, MAX_IMAGE_PIXELS), + DecompressionBombWarning, + ) def open(fp, mode="r"): @@ -2600,10 +2757,10 @@ def open(fp, mode="r"): exclusive_fp = False filename = "" - if isPath(fp): - filename = fp - elif HAS_PATHLIB and isinstance(fp, Path): + if HAS_PATHLIB and isinstance(fp, Path): filename = str(fp.resolve()) + elif isPath(fp): + filename = fp if filename: fp = builtins.open(filename, "rb") @@ -2620,6 +2777,7 @@ def open(fp, mode="r"): preinit() accept_warnings = [] + def _open_core(fp, filename, prefix): for i in ID: try: @@ -2637,6 +2795,10 @@ def open(fp, mode="r"): # opening failures that are entirely expected. # logger.debug("", exc_info=True) continue + except BaseException: + if exclusive_fp: + fp.close() + raise return None im = _open_core(fp, filename, prefix) @@ -2653,8 +2815,8 @@ def open(fp, mode="r"): fp.close() for message in accept_warnings: warnings.warn(message) - raise IOError("cannot identify image file %r" - % (filename if filename else fp)) + raise IOError("cannot identify image file %r" % (filename if filename else fp)) + # # Image processing. @@ -2758,6 +2920,7 @@ def merge(mode, bands): # -------------------------------------------------------------------- # Plugin registry + def register_open(id, factory, accept=None): """ Register an image file plugin. This function should not be used @@ -2871,6 +3034,7 @@ def register_encoder(name, encoder): # -------------------------------------------------------------------- # Simple display support. User code may override this. + def _show(image, **options): # override me, as necessary _showxv(image, **options) @@ -2878,12 +3042,14 @@ def _show(image, **options): def _showxv(image, title=None, **options): from . import ImageShow + ImageShow.show(image, title, **options) # -------------------------------------------------------------------- # Effects + def effect_mandelbrot(size, extent, quality): """ Generate a Mandelbrot set covering the given extent. @@ -2929,14 +3095,15 @@ def radial_gradient(mode): # -------------------------------------------------------------------- # Resources + def _apply_env_variables(env=None): if env is None: env = os.environ for var_name, setter in [ - ('PILLOW_ALIGNMENT', core.set_alignment), - ('PILLOW_BLOCK_SIZE', core.set_block_size), - ('PILLOW_BLOCKS_MAX', core.set_blocks_max), + ("PILLOW_ALIGNMENT", core.set_alignment), + ("PILLOW_BLOCK_SIZE", core.set_block_size), + ("PILLOW_BLOCKS_MAX", core.set_blocks_max), ]: if var_name not in env: continue @@ -2944,22 +3111,233 @@ def _apply_env_variables(env=None): var = env[var_name].lower() units = 1 - for postfix, mul in [('k', 1024), ('m', 1024*1024)]: + for postfix, mul in [("k", 1024), ("m", 1024 * 1024)]: if var.endswith(postfix): units = mul - var = var[:-len(postfix)] + var = var[: -len(postfix)] try: var = int(var) * units except ValueError: - warnings.warn("{0} is not int".format(var_name)) + warnings.warn("{} is not int".format(var_name)) continue try: setter(var) except ValueError as e: - warnings.warn("{0}: {1}".format(var_name, e)) + warnings.warn("{}: {}".format(var_name, e)) _apply_env_variables() atexit.register(core.clear_cache) + + +class Exif(MutableMapping): + endian = "<" + + def __init__(self): + self._data = {} + self._ifds = {} + self._info = None + self._loaded_exif = None + + def _fixup(self, value): + try: + if len(value) == 1 and not isinstance(value, dict): + return value[0] + except Exception: + pass + return value + + def _fixup_dict(self, src_dict): + # Helper function for _getexif() + # returns a dict with any single item tuples/lists as individual values + return {k: self._fixup(v) for k, v in src_dict.items()} + + def _get_ifd_dict(self, tag): + try: + # an offset pointer to the location of the nested embedded IFD. + # It should be a long, but may be corrupted. + self.fp.seek(self[tag]) + except (KeyError, TypeError): + pass + else: + from . import TiffImagePlugin + + info = TiffImagePlugin.ImageFileDirectory_v1(self.head) + info.load(self.fp) + return self._fixup_dict(info) + + def load(self, data): + # Extract EXIF information. This is highly experimental, + # and is likely to be replaced with something better in a future + # version. + + # The EXIF record consists of a TIFF file embedded in a JPEG + # application marker (!). + if data == self._loaded_exif: + return + self._loaded_exif = data + self._data.clear() + self._ifds.clear() + self._info = None + if not data: + return + + self.fp = io.BytesIO(data[6:]) + self.head = self.fp.read(8) + # process dictionary + from . import TiffImagePlugin + + self._info = TiffImagePlugin.ImageFileDirectory_v1(self.head) + self.endian = self._info._endian + self.fp.seek(self._info.next) + self._info.load(self.fp) + + # get EXIF extension + ifd = self._get_ifd_dict(0x8769) + if ifd: + self._data.update(ifd) + self._ifds[0x8769] = ifd + + def tobytes(self, offset=0): + from . import TiffImagePlugin + + if self.endian == "<": + head = b"II\x2A\x00\x08\x00\x00\x00" + else: + head = b"MM\x00\x2A\x00\x00\x00\x08" + ifd = TiffImagePlugin.ImageFileDirectory_v2(ifh=head) + for tag, value in self.items(): + ifd[tag] = value + return b"Exif\x00\x00" + head + ifd.tobytes(offset) + + def get_ifd(self, tag): + if tag not in self._ifds and tag in self: + if tag in [0x8825, 0xA005]: + # gpsinfo, interop + self._ifds[tag] = self._get_ifd_dict(tag) + elif tag == 0x927C: # makernote + from .TiffImagePlugin import ImageFileDirectory_v2 + + if self[0x927C][:8] == b"FUJIFILM": + exif_data = self[0x927C] + ifd_offset = i32le(exif_data[8:12]) + ifd_data = exif_data[ifd_offset:] + + makernote = {} + for i in range(0, struct.unpack(" 4: + offset, = struct.unpack("H", ifd_data[:2])[0]): + ifd_tag, typ, count, data = struct.unpack( + ">HHL4s", ifd_data[i * 12 + 2 : (i + 1) * 12 + 2] + ) + if ifd_tag == 0x1101: + # CameraInfo + offset, = struct.unpack(">L", data) + self.fp.seek(offset) + + camerainfo = {"ModelID": self.fp.read(4)} + + self.fp.read(4) + # Seconds since 2000 + camerainfo["TimeStamp"] = i32le(self.fp.read(12)) + + self.fp.read(4) + camerainfo["InternalSerialNumber"] = self.fp.read(4) + + self.fp.read(12) + parallax = self.fp.read(4) + handler = ImageFileDirectory_v2._load_dispatch[ + TiffTags.FLOAT + ][1] + camerainfo["Parallax"] = handler( + ImageFileDirectory_v2(), parallax, False + ) + + self.fp.read(4) + camerainfo["Category"] = self.fp.read(2) + + makernote = {0x1101: dict(self._fixup_dict(camerainfo))} + self._ifds[0x927C] = makernote + return self._ifds.get(tag, {}) + + def __str__(self): + if self._info is not None: + # Load all keys into self._data + for tag in self._info.keys(): + self[tag] + + return str(self._data) + + def __len__(self): + keys = set(self._data) + if self._info is not None: + keys.update(self._info) + return len(keys) + + def __getitem__(self, tag): + if self._info is not None and tag not in self._data and tag in self._info: + self._data[tag] = self._fixup(self._info[tag]) + if tag == 0x8825: + self._data[tag] = self.get_ifd(tag) + del self._info[tag] + return self._data[tag] + + def __contains__(self, tag): + return tag in self._data or (self._info is not None and tag in self._info) + + if not py3: + + def has_key(self, tag): + return tag in self + + def __setitem__(self, tag, value): + if self._info is not None and tag in self._info: + del self._info[tag] + self._data[tag] = value + + def __delitem__(self, tag): + if self._info is not None and tag in self._info: + del self._info[tag] + del self._data[tag] + + def __iter__(self): + keys = set(self._data) + if self._info is not None: + keys.update(self._info) + return iter(keys) diff --git a/server/www/packages/packages-linux/x64/PIL/ImageChops.py b/server/www/packages/packages-linux/x64/PIL/ImageChops.py index 8901673..b1f71b5 100644 --- a/server/www/packages/packages-linux/x64/PIL/ImageChops.py +++ b/server/www/packages/packages-linux/x64/PIL/ImageChops.py @@ -54,7 +54,7 @@ def invert(image): def lighter(image1, image2): """ Compares the two images, pixel by pixel, and returns a new image containing - the lighter values. + the lighter values. At least one of the images must have mode "1". .. code-block:: python @@ -70,8 +70,8 @@ def lighter(image1, image2): def darker(image1, image2): """ - Compares the two images, pixel by pixel, and returns a new image - containing the darker values. + Compares the two images, pixel by pixel, and returns a new image containing + the darker values. At least one of the images must have mode "1". .. code-block:: python @@ -88,7 +88,7 @@ def darker(image1, image2): def difference(image1, image2): """ Returns the absolute value of the pixel-by-pixel difference between the two - images. + images. At least one of the images must have mode "1". .. code-block:: python @@ -107,7 +107,8 @@ def multiply(image1, image2): Superimposes two images on top of each other. If you multiply an image with a solid black image, the result is black. If - you multiply with a solid white image, the image is unaffected. + you multiply with a solid white image, the image is unaffected. At least + one of the images must have mode "1". .. code-block:: python @@ -123,7 +124,8 @@ def multiply(image1, image2): def screen(image1, image2): """ - Superimposes two inverted images on top of each other. + Superimposes two inverted images on top of each other. At least one of the + images must have mode "1". .. code-block:: python @@ -141,6 +143,7 @@ def add(image1, image2, scale=1.0, offset=0): """ Adds two images, dividing the result by scale and adding the offset. If omitted, scale defaults to 1.0, and offset to 0.0. + At least one of the images must have mode "1". .. code-block:: python @@ -156,8 +159,9 @@ def add(image1, image2, scale=1.0, offset=0): def subtract(image1, image2, scale=1.0, offset=0): """ - Subtracts two images, dividing the result by scale and adding the - offset. If omitted, scale defaults to 1.0, and offset to 0.0. + Subtracts two images, dividing the result by scale and adding the offset. + If omitted, scale defaults to 1.0, and offset to 0.0. At least one of the + images must have mode "1". .. code-block:: python @@ -172,7 +176,8 @@ def subtract(image1, image2, scale=1.0, offset=0): def add_modulo(image1, image2): - """Add two images, without clipping the result. + """Add two images, without clipping the result. At least one of the images + must have mode "1". .. code-block:: python @@ -187,7 +192,8 @@ def add_modulo(image1, image2): def subtract_modulo(image1, image2): - """Subtract two images, without clipping the result. + """Subtract two images, without clipping the result. At least one of the + images must have mode "1". .. code-block:: python @@ -202,7 +208,8 @@ def subtract_modulo(image1, image2): def logical_and(image1, image2): - """Logical AND between two images. + """Logical AND between two images. At least one of the images must have + mode "1". .. code-block:: python @@ -217,7 +224,8 @@ def logical_and(image1, image2): def logical_or(image1, image2): - """Logical OR between two images. + """Logical OR between two images. At least one of the images must have + mode "1". .. code-block:: python @@ -232,7 +240,8 @@ def logical_or(image1, image2): def logical_xor(image1, image2): - """Logical XOR between two images. + """Logical XOR between two images. At least one of the images must have + mode "1". .. code-block:: python diff --git a/server/www/packages/packages-linux/x64/PIL/ImageCms.py b/server/www/packages/packages-linux/x64/PIL/ImageCms.py index 4b6281f..ed4eefc 100644 --- a/server/www/packages/packages-linux/x64/PIL/ImageCms.py +++ b/server/www/packages/packages-linux/x64/PIL/ImageCms.py @@ -16,17 +16,20 @@ # below for the original description. from __future__ import print_function + import sys from PIL import Image +from PIL._util import isStringType + try: from PIL import _imagingcms except ImportError as ex: # Allow error import for doc purposes, but error out when accessing # anything in core. - from _util import deferred_error + from ._util import deferred_error + _imagingcms = deferred_error(ex) -from PIL._util import isStringType DESCRIPTION = """ pyCMS @@ -132,7 +135,7 @@ FLAGS = { "SOFTPROOFING": 16384, # Do softproofing "PRESERVEBLACK": 32768, # Black preservation "NODEFAULTRESOURCEDEF": 16777216, # CRD special - "GRIDPOINTS": lambda n: ((n) & 0xFF) << 16 # Gridpoints + "GRIDPOINTS": lambda n: ((n) & 0xFF) << 16, # Gridpoints } _MAX_FLAG = 0 @@ -148,8 +151,8 @@ for flag in FLAGS.values(): ## # Profile. -class ImageCmsProfile(object): +class ImageCmsProfile(object): def __init__(self, profile): """ :param profile: Either a string representing a filename, @@ -197,22 +200,31 @@ class ImageCmsTransform(Image.ImagePointHandler): Will return the output profile in the output.info['icc_profile']. """ - def __init__(self, input, output, input_mode, output_mode, - intent=INTENT_PERCEPTUAL, proof=None, - proof_intent=INTENT_ABSOLUTE_COLORIMETRIC, flags=0): + def __init__( + self, + input, + output, + input_mode, + output_mode, + intent=INTENT_PERCEPTUAL, + proof=None, + proof_intent=INTENT_ABSOLUTE_COLORIMETRIC, + flags=0, + ): if proof is None: self.transform = core.buildTransform( - input.profile, output.profile, - input_mode, output_mode, - intent, - flags + input.profile, output.profile, input_mode, output_mode, intent, flags ) else: self.transform = core.buildProofTransform( - input.profile, output.profile, proof.profile, - input_mode, output_mode, - intent, proof_intent, - flags + input.profile, + output.profile, + proof.profile, + input_mode, + output_mode, + intent, + proof_intent, + flags, ) # Note: inputMode and outputMode are for pyCMS compatibility only self.input_mode = self.inputMode = input_mode @@ -228,7 +240,7 @@ class ImageCmsTransform(Image.ImagePointHandler): if imOut is None: imOut = Image.new(self.output_mode, im.size, None) self.transform.apply(im.im.id, imOut.im.id) - imOut.info['icc_profile'] = self.output_profile.tobytes() + imOut.info["icc_profile"] = self.output_profile.tobytes() return imOut def apply_in_place(self, im): @@ -236,7 +248,7 @@ class ImageCmsTransform(Image.ImagePointHandler): if im.mode != self.output_mode: raise ValueError("mode mismatch") # wrong output mode self.transform.apply(im.im.id, im.im.id) - im.info['icc_profile'] = self.output_profile.tobytes() + im.info["icc_profile"] = self.output_profile.tobytes() return im @@ -247,6 +259,7 @@ def get_display_profile(handle=None): if sys.platform == "win32": from PIL import ImageWin + if isinstance(handle, ImageWin.HDC): profile = core.get_display_profile_win32(handle, 1) else: @@ -265,22 +278,30 @@ def get_display_profile(handle=None): # pyCMS compatible layer # --------------------------------------------------------------------. + class PyCMSError(Exception): """ (pyCMS) Exception class. This is used for all errors in the pyCMS API. """ + pass def profileToProfile( - im, inputProfile, outputProfile, renderingIntent=INTENT_PERCEPTUAL, - outputMode=None, inPlace=0, flags=0): + im, + inputProfile, + outputProfile, + renderingIntent=INTENT_PERCEPTUAL, + outputMode=None, + inPlace=False, + flags=0, +): """ (pyCMS) Applies an ICC transformation to a given image, mapping from inputProfile to outputProfile. If the input or output profiles specified are not valid filenames, a - PyCMSError will be raised. If inPlace == TRUE and outputMode != im.mode, + PyCMSError will be raised. If inPlace is True and outputMode != im.mode, a PyCMSError will be raised. If an error occurs during application of the profiles, a PyCMSError will be raised. If outputMode is not a mode supported by the outputProfile (or by pyCMS), a PyCMSError will be @@ -317,9 +338,9 @@ def profileToProfile( MUST be the same mode as the input, or omitted completely. If omitted, the outputMode will be the same as the mode of the input image (im.mode) - :param inPlace: Boolean (1 = True, None or 0 = False). If True, the - original image is modified in-place, and None is returned. If False - (default), a new Image object is returned with the transform applied. + :param inPlace: Boolean. If True, the original image is modified in-place, + and None is returned. If False (default), a new Image object is + returned with the transform applied. :param flags: Integer (0-...) specifying additional flags :returns: Either None or a new PIL image object, depending on value of inPlace @@ -333,8 +354,7 @@ def profileToProfile( raise PyCMSError("renderingIntent must be an integer between 0 and 3") if not isinstance(flags, int) or not (0 <= flags <= _MAX_FLAG): - raise PyCMSError( - "flags must be an integer between 0 and %s" + _MAX_FLAG) + raise PyCMSError("flags must be an integer between 0 and %s" + _MAX_FLAG) try: if not isinstance(inputProfile, ImageCmsProfile): @@ -342,8 +362,12 @@ def profileToProfile( if not isinstance(outputProfile, ImageCmsProfile): outputProfile = ImageCmsProfile(outputProfile) transform = ImageCmsTransform( - inputProfile, outputProfile, im.mode, outputMode, - renderingIntent, flags=flags + inputProfile, + outputProfile, + im.mode, + outputMode, + renderingIntent, + flags=flags, ) if inPlace: transform.apply_in_place(im) @@ -379,8 +403,13 @@ def getOpenProfile(profileFilename): def buildTransform( - inputProfile, outputProfile, inMode, outMode, - renderingIntent=INTENT_PERCEPTUAL, flags=0): + inputProfile, + outputProfile, + inMode, + outMode, + renderingIntent=INTENT_PERCEPTUAL, + flags=0, +): """ (pyCMS) Builds an ICC transform mapping from the inputProfile to the outputProfile. Use applyTransform to apply the transform to a given @@ -440,8 +469,7 @@ def buildTransform( raise PyCMSError("renderingIntent must be an integer between 0 and 3") if not isinstance(flags, int) or not (0 <= flags <= _MAX_FLAG): - raise PyCMSError( - "flags must be an integer between 0 and %s" + _MAX_FLAG) + raise PyCMSError("flags must be an integer between 0 and %s" + _MAX_FLAG) try: if not isinstance(inputProfile, ImageCmsProfile): @@ -449,17 +477,22 @@ def buildTransform( if not isinstance(outputProfile, ImageCmsProfile): outputProfile = ImageCmsProfile(outputProfile) return ImageCmsTransform( - inputProfile, outputProfile, inMode, outMode, - renderingIntent, flags=flags) + inputProfile, outputProfile, inMode, outMode, renderingIntent, flags=flags + ) except (IOError, TypeError, ValueError) as v: raise PyCMSError(v) def buildProofTransform( - inputProfile, outputProfile, proofProfile, inMode, outMode, - renderingIntent=INTENT_PERCEPTUAL, - proofRenderingIntent=INTENT_ABSOLUTE_COLORIMETRIC, - flags=FLAGS["SOFTPROOFING"]): + inputProfile, + outputProfile, + proofProfile, + inMode, + outMode, + renderingIntent=INTENT_PERCEPTUAL, + proofRenderingIntent=INTENT_ABSOLUTE_COLORIMETRIC, + flags=FLAGS["SOFTPROOFING"], +): """ (pyCMS) Builds an ICC transform mapping from the inputProfile to the outputProfile, but tries to simulate the result that would be @@ -538,8 +571,7 @@ def buildProofTransform( raise PyCMSError("renderingIntent must be an integer between 0 and 3") if not isinstance(flags, int) or not (0 <= flags <= _MAX_FLAG): - raise PyCMSError( - "flags must be an integer between 0 and %s" + _MAX_FLAG) + raise PyCMSError("flags must be an integer between 0 and %s" + _MAX_FLAG) try: if not isinstance(inputProfile, ImageCmsProfile): @@ -549,8 +581,15 @@ def buildProofTransform( if not isinstance(proofProfile, ImageCmsProfile): proofProfile = ImageCmsProfile(proofProfile) return ImageCmsTransform( - inputProfile, outputProfile, inMode, outMode, renderingIntent, - proofProfile, proofRenderingIntent, flags) + inputProfile, + outputProfile, + inMode, + outMode, + renderingIntent, + proofProfile, + proofRenderingIntent, + flags, + ) except (IOError, TypeError, ValueError) as v: raise PyCMSError(v) @@ -559,16 +598,16 @@ buildTransformFromOpenProfiles = buildTransform buildProofTransformFromOpenProfiles = buildProofTransform -def applyTransform(im, transform, inPlace=0): +def applyTransform(im, transform, inPlace=False): """ (pyCMS) Applies a transform to a given image. If im.mode != transform.inMode, a PyCMSError is raised. - If inPlace == TRUE and transform.inMode != transform.outMode, a + If inPlace is True and transform.inMode != transform.outMode, a PyCMSError is raised. - If im.mode, transfer.inMode, or transfer.outMode is not supported by + If im.mode, transform.inMode, or transform.outMode is not supported by pyCMSdll or the profiles you used for the transform, a PyCMSError is raised. @@ -581,7 +620,7 @@ def applyTransform(im, transform, inPlace=0): considerable calculation time if doing the same conversion multiple times. If you want to modify im in-place instead of receiving a new image as - the return value, set inPlace to TRUE. This can only be done if + the return value, set inPlace to True. This can only be done if transform.inMode and transform.outMode are the same, because we can't change the mode in-place (the buffer sizes for some modes are different). The default behavior is to return a new Image object of @@ -590,10 +629,9 @@ def applyTransform(im, transform, inPlace=0): :param im: A PIL Image object, and im.mode must be the same as the inMode supported by the transform. :param transform: A valid CmsTransform class object - :param inPlace: Bool (1 == True, 0 or None == False). If True, im is - modified in place and None is returned, if False, a new Image object - with the transform applied is returned (and im is not changed). The - default is False. + :param inPlace: Bool. If True, im is modified in place and None is + returned, if False, a new Image object with the transform applied is + returned (and im is not changed). The default is False. :returns: Either None, or a new PIL Image object, depending on the value of inPlace. The profile will be returned in the image's info['icc_profile']. @@ -642,15 +680,16 @@ def createProfile(colorSpace, colorTemp=-1): if colorSpace not in ["LAB", "XYZ", "sRGB"]: raise PyCMSError( "Color space not supported for on-the-fly profile creation (%s)" - % colorSpace) + % colorSpace + ) if colorSpace == "LAB": try: colorTemp = float(colorTemp) - except: + except (TypeError, ValueError): raise PyCMSError( - "Color temperature must be numeric, \"%s\" not valid" - % colorTemp) + 'Color temperature must be numeric, "%s" not valid' % colorTemp + ) try: return core.createProfile(colorSpace, colorTemp) @@ -687,11 +726,11 @@ def getProfileName(profile): # // name was "%s - %s" (model, manufacturer) || Description , # // but if the Model and Manufacturer were the same or the model # // was long, Just the model, in 1.x - model = profile.profile.product_model - manufacturer = profile.profile.product_manufacturer + model = profile.profile.model + manufacturer = profile.profile.manufacturer if not (model or manufacturer): - return profile.profile.product_description + "\n" + return (profile.profile.profile_description or "") + "\n" if not manufacturer or len(model) > 30: return model + "\n" return "%s - %s\n" % (model, manufacturer) @@ -727,9 +766,9 @@ def getProfileInfo(profile): # add an extra newline to preserve pyCMS compatibility # Python, not C. the white point bits weren't working well, # so skipping. - # // info was description \r\n\r\n copyright \r\n\r\n K007 tag \r\n\r\n whitepoint - description = profile.profile.product_description - cpright = profile.profile.product_copyright + # info was description \r\n\r\n copyright \r\n\r\n K007 tag \r\n\r\n whitepoint + description = profile.profile.profile_description + cpright = profile.profile.copyright arr = [] for elt in (description, cpright): if elt: @@ -763,7 +802,7 @@ def getProfileCopyright(profile): # add an extra newline to preserve pyCMS compatibility if not isinstance(profile, ImageCmsProfile): profile = ImageCmsProfile(profile) - return profile.profile.product_copyright + "\n" + return (profile.profile.copyright or "") + "\n" except (AttributeError, IOError, TypeError, ValueError) as v: raise PyCMSError(v) @@ -791,7 +830,7 @@ def getProfileManufacturer(profile): # add an extra newline to preserve pyCMS compatibility if not isinstance(profile, ImageCmsProfile): profile = ImageCmsProfile(profile) - return profile.profile.product_manufacturer + "\n" + return (profile.profile.manufacturer or "") + "\n" except (AttributeError, IOError, TypeError, ValueError) as v: raise PyCMSError(v) @@ -820,7 +859,7 @@ def getProfileModel(profile): # add an extra newline to preserve pyCMS compatibility if not isinstance(profile, ImageCmsProfile): profile = ImageCmsProfile(profile) - return profile.profile.product_model + "\n" + return (profile.profile.model or "") + "\n" except (AttributeError, IOError, TypeError, ValueError) as v: raise PyCMSError(v) @@ -849,7 +888,7 @@ def getProfileDescription(profile): # add an extra newline to preserve pyCMS compatibility if not isinstance(profile, ImageCmsProfile): profile = ImageCmsProfile(profile) - return profile.profile.product_description + "\n" + return (profile.profile.profile_description or "") + "\n" except (AttributeError, IOError, TypeError, ValueError) as v: raise PyCMSError(v) @@ -949,7 +988,4 @@ def versions(): (pyCMS) Fetches versions. """ - return ( - VERSION, core.littlecms_version, - sys.version.split()[0], Image.VERSION - ) + return (VERSION, core.littlecms_version, sys.version.split()[0], Image.__version__) diff --git a/server/www/packages/packages-linux/x64/PIL/ImageColor.py b/server/www/packages/packages-linux/x64/PIL/ImageColor.py index 08c00fd..692d7d2 100644 --- a/server/www/packages/packages-linux/x64/PIL/ImageColor.py +++ b/server/www/packages/packages-linux/x64/PIL/ImageColor.py @@ -17,9 +17,10 @@ # See the README file for information on usage and redistribution. # -from . import Image import re +from . import Image + def getrgb(color): """ @@ -41,89 +42,77 @@ def getrgb(color): return rgb # check for known string formats - if re.match('#[a-f0-9]{3}$', color): - return ( - int(color[1]*2, 16), - int(color[2]*2, 16), - int(color[3]*2, 16), - ) + if re.match("#[a-f0-9]{3}$", color): + return (int(color[1] * 2, 16), int(color[2] * 2, 16), int(color[3] * 2, 16)) - if re.match('#[a-f0-9]{4}$', color): + if re.match("#[a-f0-9]{4}$", color): return ( - int(color[1]*2, 16), - int(color[2]*2, 16), - int(color[3]*2, 16), - int(color[4]*2, 16), - ) + int(color[1] * 2, 16), + int(color[2] * 2, 16), + int(color[3] * 2, 16), + int(color[4] * 2, 16), + ) - if re.match('#[a-f0-9]{6}$', color): - return ( - int(color[1:3], 16), - int(color[3:5], 16), - int(color[5:7], 16), - ) + if re.match("#[a-f0-9]{6}$", color): + return (int(color[1:3], 16), int(color[3:5], 16), int(color[5:7], 16)) - if re.match('#[a-f0-9]{8}$', color): + if re.match("#[a-f0-9]{8}$", color): return ( int(color[1:3], 16), int(color[3:5], 16), int(color[5:7], 16), int(color[7:9], 16), - ) + ) m = re.match(r"rgb\(\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*\)$", color) if m: - return ( - int(m.group(1)), - int(m.group(2)), - int(m.group(3)) - ) + return (int(m.group(1)), int(m.group(2)), int(m.group(3))) m = re.match(r"rgb\(\s*(\d+)%\s*,\s*(\d+)%\s*,\s*(\d+)%\s*\)$", color) if m: return ( int((int(m.group(1)) * 255) / 100.0 + 0.5), int((int(m.group(2)) * 255) / 100.0 + 0.5), - int((int(m.group(3)) * 255) / 100.0 + 0.5) - ) + int((int(m.group(3)) * 255) / 100.0 + 0.5), + ) - m = re.match(r"hsl\(\s*(\d+\.?\d*)\s*,\s*(\d+\.?\d*)%\s*,\s*(\d+\.?\d*)%\s*\)$", color) + m = re.match( + r"hsl\(\s*(\d+\.?\d*)\s*,\s*(\d+\.?\d*)%\s*,\s*(\d+\.?\d*)%\s*\)$", color + ) if m: from colorsys import hls_to_rgb + rgb = hls_to_rgb( float(m.group(1)) / 360.0, float(m.group(3)) / 100.0, float(m.group(2)) / 100.0, - ) + ) return ( int(rgb[0] * 255 + 0.5), int(rgb[1] * 255 + 0.5), - int(rgb[2] * 255 + 0.5) - ) + int(rgb[2] * 255 + 0.5), + ) - m = re.match(r"hs[bv]\(\s*(\d+\.?\d*)\s*,\s*(\d+\.?\d*)%\s*,\s*(\d+\.?\d*)%\s*\)$", color) + m = re.match( + r"hs[bv]\(\s*(\d+\.?\d*)\s*,\s*(\d+\.?\d*)%\s*,\s*(\d+\.?\d*)%\s*\)$", color + ) if m: from colorsys import hsv_to_rgb + rgb = hsv_to_rgb( float(m.group(1)) / 360.0, float(m.group(2)) / 100.0, float(m.group(3)) / 100.0, - ) + ) return ( int(rgb[0] * 255 + 0.5), int(rgb[1] * 255 + 0.5), - int(rgb[2] * 255 + 0.5) - ) + int(rgb[2] * 255 + 0.5), + ) - m = re.match(r"rgba\(\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*\)$", - color) + m = re.match(r"rgba\(\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*\)$", color) if m: - return ( - int(m.group(1)), - int(m.group(2)), - int(m.group(3)), - int(m.group(4)) - ) + return (int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4))) raise ValueError("unknown color specifier: %r" % color) @@ -145,11 +134,11 @@ def getcolor(color, mode): if Image.getmodebase(mode) == "L": r, g, b = color - color = (r*299 + g*587 + b*114)//1000 - if mode[-1] == 'A': + color = (r * 299 + g * 587 + b * 114) // 1000 + if mode[-1] == "A": return (color, alpha) else: - if mode[-1] == 'A': + if mode[-1] == "A": return color + (alpha,) return color diff --git a/server/www/packages/packages-linux/x64/PIL/ImageDraw.py b/server/www/packages/packages-linux/x64/PIL/ImageDraw.py index 6a70def..ed3383f 100644 --- a/server/www/packages/packages-linux/x64/PIL/ImageDraw.py +++ b/server/www/packages/packages-linux/x64/PIL/ImageDraw.py @@ -36,6 +36,7 @@ import numbers from . import Image, ImageColor from ._util import isStringType + """ A simple 2D drawing interface for PIL images.

@@ -45,7 +46,6 @@ directly. class ImageDraw(object): - def __init__(self, im, mode=None): """ Create a drawing instance. @@ -76,9 +76,9 @@ class ImageDraw(object): self.draw = Image.core.draw(self.im, blend) self.mode = mode if mode in ("I", "F"): - self.ink = self.draw.draw_ink(1, mode) + self.ink = self.draw.draw_ink(1) else: - self.ink = self.draw.draw_ink(-1, mode) + self.ink = self.draw.draw_ink(-1) if mode in ("1", "P", "I", "F"): # FIXME: fix Fill2 to properly support matte for I+F images self.fontmode = "1" @@ -95,6 +95,7 @@ class ImageDraw(object): if not self.font: # FIXME: should add a font repository from . import ImageFont + self.font = ImageFont.load_default() return self.font @@ -110,13 +111,13 @@ class ImageDraw(object): ink = ImageColor.getcolor(ink, self.mode) if self.palette and not isinstance(ink, numbers.Number): ink = self.palette.getcolor(ink) - ink = self.draw.draw_ink(ink, self.mode) + ink = self.draw.draw_ink(ink) if fill is not None: if isStringType(fill): fill = ImageColor.getcolor(fill, self.mode) if self.palette and not isinstance(fill, numbers.Number): fill = self.palette.getcolor(fill) - fill = self.draw.draw_ink(fill, self.mode) + fill = self.draw.draw_ink(fill) return ink, fill def arc(self, xy, start, end, fill=None, width=0): @@ -156,13 +157,12 @@ class ImageDraw(object): if ink is not None: self.draw.draw_lines(xy, ink, width) if joint == "curve" and width > 4: - for i in range(1, len(xy)-1): + for i in range(1, len(xy) - 1): point = xy[i] angles = [ - math.degrees(math.atan2( - end[0] - start[0], start[1] - end[1] - )) % 360 - for start, end in ((xy[i-1], point), (point, xy[i+1])) + math.degrees(math.atan2(end[0] - start[0], start[1] - end[1])) + % 360 + for start, end in ((xy[i - 1], point), (point, xy[i + 1])) ] if angles[0] == angles[1]: # This is a straight line, so no joint is required @@ -171,21 +171,23 @@ class ImageDraw(object): def coord_at_angle(coord, angle): x, y = coord angle -= 90 - distance = width/2 - 1 - return tuple([ - p + - (math.floor(p_d) if p_d > 0 else math.ceil(p_d)) - for p, p_d in - ((x, distance * math.cos(math.radians(angle))), - (y, distance * math.sin(math.radians(angle)))) - ]) - flipped = ((angles[1] > angles[0] and - angles[1] - 180 > angles[0]) or - (angles[1] < angles[0] and - angles[1] + 180 > angles[0])) + distance = width / 2 - 1 + return tuple( + [ + p + (math.floor(p_d) if p_d > 0 else math.ceil(p_d)) + for p, p_d in ( + (x, distance * math.cos(math.radians(angle))), + (y, distance * math.sin(math.radians(angle))), + ) + ] + ) + + flipped = ( + angles[1] > angles[0] and angles[1] - 180 > angles[0] + ) or (angles[1] < angles[0] and angles[1] + 180 > angles[0]) coords = [ - (point[0] - width/2 + 1, point[1] - width/2 + 1), - (point[0] + width/2 - 1, point[1] + width/2 - 1) + (point[0] - width / 2 + 1, point[1] - width / 2 + 1), + (point[0] + width / 2 - 1, point[1] + width / 2 - 1), ] if flipped: start, end = (angles[1] + 90, angles[0] + 90) @@ -197,15 +199,15 @@ class ImageDraw(object): # Cover potential gaps between the line and the joint if flipped: gapCoords = [ - coord_at_angle(point, angles[0]+90), + coord_at_angle(point, angles[0] + 90), point, - coord_at_angle(point, angles[1]+90) + coord_at_angle(point, angles[1] + 90), ] else: gapCoords = [ - coord_at_angle(point, angles[0]-90), + coord_at_angle(point, angles[0] - 90), point, - coord_at_angle(point, angles[1]-90) + coord_at_angle(point, angles[1] - 90), ] self.line(gapCoords, fill, width=3) @@ -259,36 +261,126 @@ class ImageDraw(object): return text.split(split_character) - def text(self, xy, text, fill=None, font=None, anchor=None, - *args, **kwargs): + def text( + self, + xy, + text, + fill=None, + font=None, + anchor=None, + spacing=4, + align="left", + direction=None, + features=None, + language=None, + stroke_width=0, + stroke_fill=None, + *args, + **kwargs + ): if self._multiline_check(text): - return self.multiline_text(xy, text, fill, font, anchor, - *args, **kwargs) - ink, fill = self._getink(fill) + return self.multiline_text( + xy, + text, + fill, + font, + anchor, + spacing, + align, + direction, + features, + language, + stroke_width, + stroke_fill, + ) + if font is None: font = self.getfont() - if ink is None: - ink = fill - if ink is not None: + + def getink(fill): + ink, fill = self._getink(fill) + if ink is None: + return fill + return ink + + def draw_text(ink, stroke_width=0, stroke_offset=None): + coord = xy try: - mask, offset = font.getmask2(text, self.fontmode, - *args, **kwargs) - xy = xy[0] + offset[0], xy[1] + offset[1] + mask, offset = font.getmask2( + text, + self.fontmode, + direction=direction, + features=features, + language=language, + stroke_width=stroke_width, + *args, + **kwargs + ) + coord = coord[0] + offset[0], coord[1] + offset[1] except AttributeError: try: - mask = font.getmask(text, self.fontmode, *args, **kwargs) + mask = font.getmask( + text, + self.fontmode, + direction, + features, + language, + stroke_width, + *args, + **kwargs + ) except TypeError: mask = font.getmask(text) - self.draw.draw_bitmap(xy, mask, ink) + if stroke_offset: + coord = coord[0] + stroke_offset[0], coord[1] + stroke_offset[1] + self.draw.draw_bitmap(coord, mask, ink) - def multiline_text(self, xy, text, fill=None, font=None, anchor=None, - spacing=4, align="left", direction=None, features=None): + ink = getink(fill) + if ink is not None: + stroke_ink = None + if stroke_width: + stroke_ink = getink(stroke_fill) if stroke_fill is not None else ink + + if stroke_ink is not None: + # Draw stroked text + draw_text(stroke_ink, stroke_width) + + # Draw normal text + draw_text(ink, 0, (stroke_width, stroke_width)) + else: + # Only draw normal text + draw_text(ink) + + def multiline_text( + self, + xy, + text, + fill=None, + font=None, + anchor=None, + spacing=4, + align="left", + direction=None, + features=None, + language=None, + stroke_width=0, + stroke_fill=None, + ): widths = [] max_width = 0 lines = self._multiline_split(text) - line_spacing = self.textsize('A', font=font)[1] + spacing + line_spacing = ( + self.textsize("A", font=font, stroke_width=stroke_width)[1] + spacing + ) for line in lines: - line_width, line_height = self.textsize(line, font) + line_width, line_height = self.textsize( + line, + font, + direction=direction, + features=features, + language=language, + stroke_width=stroke_width, + ) widths.append(line_width) max_width = max(max_width, line_width) left, top = xy @@ -298,35 +390,65 @@ class ImageDraw(object): elif align == "center": left += (max_width - widths[idx]) / 2.0 elif align == "right": - left += (max_width - widths[idx]) + left += max_width - widths[idx] else: raise ValueError('align must be "left", "center" or "right"') - self.text((left, top), line, fill, font, anchor, - direction=direction, features=features) + self.text( + (left, top), + line, + fill, + font, + anchor, + direction=direction, + features=features, + language=language, + stroke_width=stroke_width, + stroke_fill=stroke_fill, + ) top += line_spacing left = xy[0] - def textsize(self, text, font=None, spacing=4, direction=None, - features=None): + def textsize( + self, + text, + font=None, + spacing=4, + direction=None, + features=None, + language=None, + stroke_width=0, + ): """Get the size of a given string, in pixels.""" if self._multiline_check(text): - return self.multiline_textsize(text, font, spacing, - direction, features) + return self.multiline_textsize( + text, font, spacing, direction, features, language, stroke_width + ) if font is None: font = self.getfont() - return font.getsize(text, direction, features) + return font.getsize(text, direction, features, language, stroke_width) - def multiline_textsize(self, text, font=None, spacing=4, direction=None, - features=None): + def multiline_textsize( + self, + text, + font=None, + spacing=4, + direction=None, + features=None, + language=None, + stroke_width=0, + ): max_width = 0 lines = self._multiline_split(text) - line_spacing = self.textsize('A', font=font)[1] + spacing + line_spacing = ( + self.textsize("A", font=font, stroke_width=stroke_width)[1] + spacing + ) for line in lines: - line_width, line_height = self.textsize(line, font, spacing, - direction, features) + line_width, line_height = self.textsize( + line, font, spacing, direction, features, language, stroke_width + ) max_width = max(max_width, line_width) - return max_width, len(lines)*line_spacing - spacing + return max_width, len(lines) * line_spacing - spacing def Draw(im, mode=None): @@ -391,8 +513,8 @@ def floodfill(image, xy, value, border=None, thresh=0): pixel. :param thresh: Optional threshold value which specifies a maximum tolerable difference of a pixel value from the 'background' in - order for it to be replaced. Useful for filling regions of non- - homogeneous, but similar, colors. + order for it to be replaced. Useful for filling regions of + non-homogeneous, but similar, colors. """ # based on an implementation by Eric S. Raymond # amended by yo1995 @20180806 @@ -406,13 +528,16 @@ def floodfill(image, xy, value, border=None, thresh=0): except (ValueError, IndexError): return # seed point outside image edge = {(x, y)} - full_edge = set() # use a set to keep record of current and previous edge pixels to reduce memory consumption + # use a set to keep record of current and previous edge pixels + # to reduce memory consumption + full_edge = set() while edge: new_edge = set() for (x, y) in edge: # 4 adjacent method - for (s, t) in ((x+1, y), (x-1, y), (x, y+1), (x, y-1)): - if (s, t) in full_edge: - continue # if already processed, skip + for (s, t) in ((x + 1, y), (x - 1, y), (x, y + 1), (x, y - 1)): + # If already processed, or if a coordinate is negative, skip + if (s, t) in full_edge or s < 0 or t < 0: + continue try: p = pixel[s, t] except (ValueError, IndexError): @@ -435,6 +560,6 @@ def _color_diff(color1, color2): Uses 1-norm distance to calculate difference between two values. """ if isinstance(color2, tuple): - return sum([abs(color1[i]-color2[i]) for i in range(0, len(color2))]) + return sum([abs(color1[i] - color2[i]) for i in range(0, len(color2))]) else: - return abs(color1-color2) + return abs(color1 - color2) diff --git a/server/www/packages/packages-linux/x64/PIL/ImageDraw2.py b/server/www/packages/packages-linux/x64/PIL/ImageDraw2.py index f7902b0..324d869 100644 --- a/server/www/packages/packages-linux/x64/PIL/ImageDraw2.py +++ b/server/www/packages/packages-linux/x64/PIL/ImageDraw2.py @@ -38,7 +38,6 @@ class Font(object): class Draw(object): - def __init__(self, image, size=None, color=None): if not hasattr(image, "im"): image = Image.new(image, size, color) diff --git a/server/www/packages/packages-linux/x64/PIL/ImageEnhance.py b/server/www/packages/packages-linux/x64/PIL/ImageEnhance.py index 1b78bfd..534eb4f 100644 --- a/server/www/packages/packages-linux/x64/PIL/ImageEnhance.py +++ b/server/www/packages/packages-linux/x64/PIL/ImageEnhance.py @@ -22,7 +22,6 @@ from . import Image, ImageFilter, ImageStat class _Enhance(object): - def enhance(self, factor): """ Returns an enhanced image. @@ -45,14 +44,14 @@ class Color(_Enhance): factor of 0.0 gives a black and white image. A factor of 1.0 gives the original image. """ + def __init__(self, image): self.image = image - self.intermediate_mode = 'L' - if 'A' in image.getbands(): - self.intermediate_mode = 'LA' + self.intermediate_mode = "L" + if "A" in image.getbands(): + self.intermediate_mode = "LA" - self.degenerate = image.convert( - self.intermediate_mode).convert(image.mode) + self.degenerate = image.convert(self.intermediate_mode).convert(image.mode) class Contrast(_Enhance): @@ -62,13 +61,14 @@ class Contrast(_Enhance): to the contrast control on a TV set. An enhancement factor of 0.0 gives a solid grey image. A factor of 1.0 gives the original image. """ + def __init__(self, image): self.image = image mean = int(ImageStat.Stat(image.convert("L")).mean[0] + 0.5) self.degenerate = Image.new("L", image.size, mean).convert(image.mode) - if 'A' in image.getbands(): - self.degenerate.putalpha(image.getchannel('A')) + if "A" in image.getbands(): + self.degenerate.putalpha(image.getchannel("A")) class Brightness(_Enhance): @@ -78,12 +78,13 @@ class Brightness(_Enhance): enhancement factor of 0.0 gives a black image. A factor of 1.0 gives the original image. """ + def __init__(self, image): self.image = image self.degenerate = Image.new(image.mode, image.size, 0) - if 'A' in image.getbands(): - self.degenerate.putalpha(image.getchannel('A')) + if "A" in image.getbands(): + self.degenerate.putalpha(image.getchannel("A")) class Sharpness(_Enhance): @@ -93,9 +94,10 @@ class Sharpness(_Enhance): enhancement factor of 0.0 gives a blurred image, a factor of 1.0 gives the original image, and a factor of 2.0 gives a sharpened image. """ + def __init__(self, image): self.image = image self.degenerate = image.filter(ImageFilter.SMOOTH) - if 'A' in image.getbands(): - self.degenerate.putalpha(image.getchannel('A')) + if "A" in image.getbands(): + self.degenerate.putalpha(image.getchannel("A")) diff --git a/server/www/packages/packages-linux/x64/PIL/ImageFile.py b/server/www/packages/packages-linux/x64/PIL/ImageFile.py index 915557a..836e631 100644 --- a/server/www/packages/packages-linux/x64/PIL/ImageFile.py +++ b/server/www/packages/packages-linux/x64/PIL/ImageFile.py @@ -27,15 +27,16 @@ # See the README file for information on usage and redistribution. # +import io +import struct +import sys + from . import Image from ._util import isPath -import io -import sys -import struct MAXBLOCK = 65536 -SAFEBLOCK = 1024*1024 +SAFEBLOCK = 1024 * 1024 LOAD_TRUNCATED_IMAGES = False @@ -44,7 +45,7 @@ ERRORS = { -2: "decoding error", -3: "unknown error", -8: "bad configuration", - -9: "out of memory error" + -9: "out of memory error", } @@ -62,6 +63,7 @@ def raise_ioerror(error): # -------------------------------------------------------------------- # Helpers + def _tilesort(t): # sort on offset return t[2] @@ -71,6 +73,7 @@ def _tilesort(t): # -------------------------------------------------------------------- # ImageFile base class + class ImageFile(Image.Image): "Base class for image file format handlers." @@ -79,6 +82,8 @@ class ImageFile(Image.Image): self._min_frame = 0 + self.custom_mimetype = None + self.tile = None self.readonly = 1 # until we know better @@ -99,11 +104,13 @@ class ImageFile(Image.Image): try: self._open() - except (IndexError, # end of data - TypeError, # end of data (ord) - KeyError, # unsupported mode - EOFError, # got header but not the first frame - struct.error) as v: + except ( + IndexError, # end of data + TypeError, # end of data (ord) + KeyError, # unsupported mode + EOFError, # got header but not the first frame + struct.error, + ) as v: # close the file only if we have opened it this constructor if self._exclusive_fp: self.fp.close() @@ -113,17 +120,18 @@ class ImageFile(Image.Image): raise SyntaxError("not identified by this driver") def draft(self, mode, size): - "Set draft mode" + """Set draft mode""" pass def get_format_mimetype(self): - if self.format is None: - return - return Image.MIME.get(self.format.upper()) + if self.custom_mimetype: + return self.custom_mimetype + if self.format is not None: + return Image.MIME.get(self.format.upper()) def verify(self): - "Check file integrity" + """Check file integrity""" # raise exception if something's wrong. must be called # directly after open, and closes file when finished. @@ -132,7 +140,7 @@ class ImageFile(Image.Image): self.fp = None def load(self): - "Load image data based on tile list" + """Load image data based on tile list""" pixel = Image.Image.load(self) @@ -144,7 +152,7 @@ class ImageFile(Image.Image): self.map = None use_mmap = self.filename and len(self.tile) == 1 # As of pypy 2.1.0, memory mapping was failing here. - use_mmap = use_mmap and not hasattr(sys, 'pypy_version_info') + use_mmap = use_mmap and not hasattr(sys, "pypy_version_info") readonly = 0 @@ -165,9 +173,12 @@ class ImageFile(Image.Image): if use_mmap: # try memory mapping decoder_name, extents, offset, args = self.tile[0] - if decoder_name == "raw" and len(args) >= 3 and \ - args[0] == self.mode and \ - args[0] in Image._MAPMODES: + if ( + decoder_name == "raw" + and len(args) >= 3 + and args[0] == self.mode + and args[0] in Image._MAPMODES + ): try: if hasattr(Image.core, "map"): # use built-in mapper WIN32 only @@ -175,16 +186,18 @@ class ImageFile(Image.Image): self.map.seek(offset) self.im = self.map.readimage( self.mode, self.size, args[1], args[2] - ) + ) else: # use mmap, if possible import mmap + with open(self.filename, "r") as fp: - self.map = mmap.mmap(fp.fileno(), 0, - access=mmap.ACCESS_READ) + self.map = mmap.mmap( + fp.fileno(), 0, access=mmap.ACCESS_READ + ) self.im = Image.core.map_buffer( - self.map, self.size, decoder_name, extents, - offset, args) + self.map, self.size, decoder_name, extents, offset, args + ) readonly = 1 # After trashing self.im, # we might need to reload the palette data. @@ -206,8 +219,9 @@ class ImageFile(Image.Image): prefix = b"" for decoder_name, extents, offset, args in self.tile: - decoder = Image._getdecoder(self.mode, decoder_name, - args, self.decoderconfig) + decoder = Image._getdecoder( + self.mode, decoder_name, args, self.decoderconfig + ) try: seek(offset) decoder.setimage(self.im, extents) @@ -230,10 +244,10 @@ class ImageFile(Image.Image): if LOAD_TRUNCATED_IMAGES: break else: - self.tile = [] - raise IOError("image file is truncated " - "(%d bytes not processed)" % - len(b)) + raise IOError( + "image file is truncated " + "(%d bytes not processed)" % len(b) + ) b = b + s n, err_code = decoder.decode(b) @@ -261,8 +275,7 @@ class ImageFile(Image.Image): def load_prepare(self): # create image memory if necessary - if not self.im or\ - self.im.mode != self.mode or self.im.size != self.size: + if not self.im or self.im.mode != self.mode or self.im.size != self.size: self.im = Image.core.new(self.mode, self.size) # create palette (optional) if self.mode == "P": @@ -281,11 +294,15 @@ class ImageFile(Image.Image): # pass def _seek_check(self, frame): - if (frame < self._min_frame or + if ( + frame < self._min_frame # Only check upper limit on frames if additional seek operations # are not required to do so - (not (hasattr(self, "_n_frames") and self._n_frames is None) and - frame >= self.n_frames+self._min_frame)): + or ( + not (hasattr(self, "_n_frames") and self._n_frames is None) + and frame >= self.n_frames + self._min_frame + ) + ): raise EOFError("attempt to seek outside sequence") return self.tell() != frame @@ -300,9 +317,7 @@ class StubImageFile(ImageFile): """ def _open(self): - raise NotImplementedError( - "StubImageFile subclass must implement _open" - ) + raise NotImplementedError("StubImageFile subclass must implement _open") def load(self): loader = self._load() @@ -315,10 +330,8 @@ class StubImageFile(ImageFile): self.__dict__ = image.__dict__ def _load(self): - "(Hook) Find actual image loader." - raise NotImplementedError( - "StubImageFile subclass must implement _load" - ) + """(Hook) Find actual image loader.""" + raise NotImplementedError("StubImageFile subclass must implement _load") class Parser(object): @@ -326,6 +339,7 @@ class Parser(object): Incremental image parser. This class implements the standard feed/close consumer interface. """ + incremental = None image = None data = None @@ -410,15 +424,13 @@ class Parser(object): im.load_prepare() d, e, o, a = im.tile[0] im.tile = [] - self.decoder = Image._getdecoder( - im.mode, d, a, im.decoderconfig - ) + self.decoder = Image._getdecoder(im.mode, d, a, im.decoderconfig) self.decoder.setimage(im.im, e) # calculate decoder offset self.offset = o if self.offset <= len(self.data): - self.data = self.data[self.offset:] + self.data = self.data[self.offset :] self.offset = 0 self.image = im @@ -460,6 +472,7 @@ class Parser(object): # -------------------------------------------------------------------- + def _save(im, fp, tile, bufsize=0): """Helper to save image based on tile list @@ -489,7 +502,7 @@ def _save(im, fp, tile, bufsize=0): for e, b, o, a in tile: e = Image._getencoder(im.mode, e, a, im.encoderconfig) if o > 0: - fp.seek(o, 0) + fp.seek(o) e.setimage(im.im, b) if e.pushes_fd: e.setfd(fp) @@ -508,7 +521,7 @@ def _save(im, fp, tile, bufsize=0): for e, b, o, a in tile: e = Image._getencoder(im.mode, e, a, im.encoderconfig) if o > 0: - fp.seek(o, 0) + fp.seek(o) e.setimage(im.im, b) if e.pushes_fd: e.setfd(fp) @@ -554,8 +567,7 @@ class PyCodecState(object): self.yoff = 0 def extents(self): - return (self.xoff, self.yoff, - self.xoff+self.xsize, self.yoff+self.ysize) + return (self.xoff, self.yoff, self.xoff + self.xsize, self.yoff + self.ysize) class PyDecoder(object): @@ -593,8 +605,6 @@ class PyDecoder(object): Override to perform the decoding process. :param buffer: A bytes object with the data to be decoded. - If `handles_eof` is set, then `buffer` will be empty and `self.fd` - will be set. :returns: A tuple of (bytes consumed, errcode). If finished with decoding return <0 for the bytes consumed. Err codes are from `ERRORS` @@ -647,8 +657,10 @@ class PyDecoder(object): if self.state.xsize <= 0 or self.state.ysize <= 0: raise ValueError("Size cannot be negative") - if (self.state.xsize + self.state.xoff > self.im.size[0] or - self.state.ysize + self.state.yoff > self.im.size[1]): + if ( + self.state.xsize + self.state.xoff > self.im.size[0] + or self.state.ysize + self.state.yoff > self.im.size[1] + ): raise ValueError("Tile cannot extend outside image") def set_as_raw(self, data, rawmode=None): @@ -663,7 +675,7 @@ class PyDecoder(object): if not rawmode: rawmode = self.mode - d = Image._getdecoder(self.mode, 'raw', (rawmode)) + d = Image._getdecoder(self.mode, "raw", (rawmode)) d.setimage(self.im, self.state.extents()) s = d.decode(data) diff --git a/server/www/packages/packages-linux/x64/PIL/ImageFilter.py b/server/www/packages/packages-linux/x64/PIL/ImageFilter.py index de99e64..fa4162b 100644 --- a/server/www/packages/packages-linux/x64/PIL/ImageFilter.py +++ b/server/www/packages/packages-linux/x64/PIL/ImageFilter.py @@ -57,12 +57,13 @@ class Kernel(BuiltinFilter): :param offset: Offset. If given, this value is added to the result, after it has been divided by the scale factor. """ + name = "Kernel" def __init__(self, size, kernel, scale=None, offset=0): if scale is None: # default scale is sum of kernel - scale = functools.reduce(lambda a, b: a+b, kernel) + scale = functools.reduce(lambda a, b: a + b, kernel) if size[0] * size[1] != len(kernel): raise ValueError("not enough coefficients in kernel") self.filterargs = size, scale, offset, kernel @@ -78,6 +79,7 @@ class RankFilter(Filter): ``size * size / 2`` for a median filter, ``size * size - 1`` for a max filter, etc. """ + name = "Rank" def __init__(self, size, rank): @@ -87,7 +89,7 @@ class RankFilter(Filter): def filter(self, image): if image.mode == "P": raise ValueError("cannot filter palette images") - image = image.expand(self.size//2, self.size//2) + image = image.expand(self.size // 2, self.size // 2) return image.rankfilter(self.size, self.rank) @@ -98,11 +100,12 @@ class MedianFilter(RankFilter): :param size: The kernel size, in pixels. """ + name = "Median" def __init__(self, size=3): self.size = size - self.rank = size*size//2 + self.rank = size * size // 2 class MinFilter(RankFilter): @@ -112,6 +115,7 @@ class MinFilter(RankFilter): :param size: The kernel size, in pixels. """ + name = "Min" def __init__(self, size=3): @@ -126,11 +130,12 @@ class MaxFilter(RankFilter): :param size: The kernel size, in pixels. """ + name = "Max" def __init__(self, size=3): self.size = size - self.rank = size*size-1 + self.rank = size * size - 1 class ModeFilter(Filter): @@ -141,6 +146,7 @@ class ModeFilter(Filter): :param size: The kernel size, in pixels. """ + name = "Mode" def __init__(self, size=3): @@ -155,6 +161,7 @@ class GaussianBlur(MultibandFilter): :param radius: Blur radius. """ + name = "GaussianBlur" def __init__(self, radius=2): @@ -175,6 +182,7 @@ class BoxBlur(MultibandFilter): returns an identical image. Radius 1 takes 1 pixel in each direction, i.e. 9 pixels in total. """ + name = "BoxBlur" def __init__(self, radius): @@ -197,7 +205,8 @@ class UnsharpMask(MultibandFilter): .. _digital unsharp masking: https://en.wikipedia.org/wiki/Unsharp_masking#Digital_unsharp_masking - """ + """ # noqa: E501 + name = "UnsharpMask" def __init__(self, radius=2, percent=150, threshold=3): @@ -211,96 +220,116 @@ class UnsharpMask(MultibandFilter): class BLUR(BuiltinFilter): name = "Blur" + # fmt: off filterargs = (5, 5), 16, 0, ( - 1, 1, 1, 1, 1, - 1, 0, 0, 0, 1, - 1, 0, 0, 0, 1, - 1, 0, 0, 0, 1, - 1, 1, 1, 1, 1 - ) + 1, 1, 1, 1, 1, + 1, 0, 0, 0, 1, + 1, 0, 0, 0, 1, + 1, 0, 0, 0, 1, + 1, 1, 1, 1, 1, + ) + # fmt: on class CONTOUR(BuiltinFilter): name = "Contour" + # fmt: off filterargs = (3, 3), 1, 255, ( -1, -1, -1, -1, 8, -1, - -1, -1, -1 - ) + -1, -1, -1, + ) + # fmt: on class DETAIL(BuiltinFilter): name = "Detail" + # fmt: off filterargs = (3, 3), 6, 0, ( - 0, -1, 0, + 0, -1, 0, -1, 10, -1, - 0, -1, 0 - ) + 0, -1, 0, + ) + # fmt: on class EDGE_ENHANCE(BuiltinFilter): name = "Edge-enhance" + # fmt: off filterargs = (3, 3), 2, 0, ( -1, -1, -1, -1, 10, -1, - -1, -1, -1 - ) + -1, -1, -1, + ) + # fmt: on class EDGE_ENHANCE_MORE(BuiltinFilter): name = "Edge-enhance More" + # fmt: off filterargs = (3, 3), 1, 0, ( -1, -1, -1, -1, 9, -1, - -1, -1, -1 - ) + -1, -1, -1, + ) + # fmt: on class EMBOSS(BuiltinFilter): name = "Emboss" + # fmt: off filterargs = (3, 3), 1, 128, ( - -1, 0, 0, - 0, 1, 0, - 0, 0, 0 - ) + -1, 0, 0, + 0, 1, 0, + 0, 0, 0, + ) + # fmt: on class FIND_EDGES(BuiltinFilter): name = "Find Edges" + # fmt: off filterargs = (3, 3), 1, 0, ( -1, -1, -1, -1, 8, -1, - -1, -1, -1 - ) + -1, -1, -1, + ) + # fmt: on class SHARPEN(BuiltinFilter): name = "Sharpen" + # fmt: off filterargs = (3, 3), 16, 0, ( -2, -2, -2, -2, 32, -2, - -2, -2, -2 - ) + -2, -2, -2, + ) + # fmt: on class SMOOTH(BuiltinFilter): name = "Smooth" + # fmt: off filterargs = (3, 3), 13, 0, ( - 1, 1, 1, - 1, 5, 1, - 1, 1, 1 - ) + 1, 1, 1, + 1, 5, 1, + 1, 1, 1, + ) + # fmt: on class SMOOTH_MORE(BuiltinFilter): name = "Smooth More" + # fmt: off filterargs = (5, 5), 100, 0, ( - 1, 1, 1, 1, 1, - 1, 5, 5, 5, 1, - 1, 5, 44, 5, 1, - 1, 5, 5, 5, 1, - 1, 1, 1, 1, 1 - ) + 1, 1, 1, 1, 1, + 1, 5, 5, 5, 1, + 1, 5, 44, 5, 1, + 1, 5, 5, 5, 1, + 1, 1, 1, 1, 1, + ) + # fmt: on class Color3DLUT(MultibandFilter): @@ -327,6 +356,7 @@ class Color3DLUT(MultibandFilter): than ``channels`` channels. Default is ``None``, which means that mode wouldn't be changed. """ + name = "Color 3D LUT" def __init__(self, size, table, channels=3, target_mode=None, **kwargs): @@ -338,7 +368,7 @@ class Color3DLUT(MultibandFilter): # Hidden flag `_copy_table=False` could be used to avoid extra copying # of the table if the table is specially made for the constructor. - copy_table = kwargs.get('_copy_table', True) + copy_table = kwargs.get("_copy_table", True) items = size[0] * size[1] * size[2] wrong_size = False @@ -346,8 +376,11 @@ class Color3DLUT(MultibandFilter): if copy_table: table = table.copy() - if table.shape in [(items * channels,), (items, channels), - (size[2], size[1], size[0], channels)]: + if table.shape in [ + (items * channels,), + (items, channels), + (size[2], size[1], size[0], channels), + ]: table = table.reshape(items * channels) else: wrong_size = True @@ -363,7 +396,8 @@ class Color3DLUT(MultibandFilter): if len(pixel) != channels: raise ValueError( "The elements of the table should " - "have a length of {}.".format(channels)) + "have a length of {}.".format(channels) + ) table.extend(pixel) if wrong_size or len(table) != items * channels: @@ -371,7 +405,9 @@ class Color3DLUT(MultibandFilter): "The table should have either channels * size**3 float items " "or size**3 items of channels-sized tuples with floats. " "Table should be: {}x{}x{}x{}. Actual length: {}".format( - channels, size[0], size[1], size[2], len(table))) + channels, size[0], size[1], size[2], len(table) + ) + ) self.table = table @staticmethod @@ -379,8 +415,9 @@ class Color3DLUT(MultibandFilter): try: _, _, _ = size except ValueError: - raise ValueError("Size should be either an integer or " - "a tuple of three integers.") + raise ValueError( + "Size should be either an integer or a tuple of three integers." + ) except TypeError: size = (size, size, size) size = [int(x) for x in size] @@ -411,15 +448,20 @@ class Color3DLUT(MultibandFilter): for b in range(size3D): for g in range(size2D): for r in range(size1D): - table[idx_out:idx_out + channels] = callback( - r / (size1D-1), g / (size2D-1), b / (size3D-1)) + table[idx_out : idx_out + channels] = callback( + r / (size1D - 1), g / (size2D - 1), b / (size3D - 1) + ) idx_out += channels - return cls((size1D, size2D, size3D), table, channels=channels, - target_mode=target_mode, _copy_table=False) + return cls( + (size1D, size2D, size3D), + table, + channels=channels, + target_mode=target_mode, + _copy_table=False, + ) - def transform(self, callback, with_normals=False, channels=None, - target_mode=None): + def transform(self, callback, with_normals=False, channels=None, target_mode=None): """Transforms the table values using provided callback and returns a new LUT with altered values. @@ -450,24 +492,31 @@ class Color3DLUT(MultibandFilter): for b in range(size3D): for g in range(size2D): for r in range(size1D): - values = self.table[idx_in:idx_in + ch_in] + values = self.table[idx_in : idx_in + ch_in] if with_normals: - values = callback(r / (size1D-1), g / (size2D-1), - b / (size3D-1), *values) + values = callback( + r / (size1D - 1), + g / (size2D - 1), + b / (size3D - 1), + *values + ) else: values = callback(*values) - table[idx_out:idx_out + ch_out] = values + table[idx_out : idx_out + ch_out] = values idx_in += ch_in idx_out += ch_out - return type(self)(self.size, table, channels=ch_out, - target_mode=target_mode or self.mode, - _copy_table=False) + return type(self)( + self.size, + table, + channels=ch_out, + target_mode=target_mode or self.mode, + _copy_table=False, + ) def __repr__(self): r = [ - "{} from {}".format(self.__class__.__name__, - self.table.__class__.__name__), + "{} from {}".format(self.__class__.__name__, self.table.__class__.__name__), "size={:d}x{:d}x{:d}".format(*self.size), "channels={:d}".format(self.channels), ] @@ -479,5 +528,11 @@ class Color3DLUT(MultibandFilter): from . import Image return image.color_lut_3d( - self.mode or image.mode, Image.LINEAR, self.channels, - self.size[0], self.size[1], self.size[2], self.table) + self.mode or image.mode, + Image.LINEAR, + self.channels, + self.size[0], + self.size[1], + self.size[2], + self.table, + ) diff --git a/server/www/packages/packages-linux/x64/PIL/ImageFont.py b/server/www/packages/packages-linux/x64/PIL/ImageFont.py index 5384a72..5cce9af 100644 --- a/server/www/packages/packages-linux/x64/PIL/ImageFont.py +++ b/server/www/packages/packages-linux/x64/PIL/ImageFont.py @@ -25,11 +25,12 @@ # See the README file for information on usage and redistribution. # -from . import Image -from ._util import isDirectory, isPath, py3 import os import sys +from . import Image +from ._util import isDirectory, isPath, py3 + LAYOUT_BASIC = 0 LAYOUT_RAQM = 1 @@ -72,7 +73,7 @@ class ImageFont(object): try: fullname = os.path.splitext(filename)[0] + ext image = Image.open(fullname) - except: + except Exception: pass else: if image and image.mode in ("1", "L"): @@ -98,7 +99,7 @@ class ImageFont(object): self.info.append(s) # read PILfont metrics - data = file.read(256*20) + data = file.read(256 * 20) # check image if image.mode not in ("1", "L"): @@ -109,9 +110,33 @@ class ImageFont(object): self.font = Image.core.font(image.im, data) def getsize(self, text, *args, **kwargs): + """ + Returns width and height (in pixels) of given text. + + :param text: Text to measure. + + :return: (width, height) + """ return self.font.getsize(text) def getmask(self, text, mode="", *args, **kwargs): + """ + Create a bitmap for the text. + + If the font uses antialiasing, the bitmap should have mode ``L`` and use a + maximum value of 255. Otherwise, it should have mode ``1``. + + :param text: Text to render. + :param mode: Used by some graphics drivers to indicate what mode the + driver prefers; if empty, the renderer may return either + mode. Note that the mode is always a string, to simplify + C-level implementations. + + .. versionadded:: 1.1.5 + + :return: An internal PIL storage memory instance as defined by the + :py:mod:`PIL.Image.core` interface module. + """ return self.font.getmask(text, mode) @@ -119,11 +144,11 @@ class ImageFont(object): # Wrapper for FreeType fonts. Application code should use the # truetype factory function to create font objects. + class FreeTypeFont(object): "FreeType font wrapper (requires _imagingft service)" - def __init__(self, font=None, size=10, index=0, encoding="", - layout_engine=None): + def __init__(self, font=None, size=10, index=0, encoding="", layout_engine=None): # FIXME: use service provider instead self.path = font @@ -135,60 +160,317 @@ class FreeTypeFont(object): layout_engine = LAYOUT_BASIC if core.HAVE_RAQM: layout_engine = LAYOUT_RAQM - if layout_engine == LAYOUT_RAQM and not core.HAVE_RAQM: + elif layout_engine == LAYOUT_RAQM and not core.HAVE_RAQM: layout_engine = LAYOUT_BASIC self.layout_engine = layout_engine - if isPath(font): - self.font = core.getfont(font, size, index, encoding, - layout_engine=layout_engine) - else: - self.font_bytes = font.read() + def load_from_bytes(f): + self.font_bytes = f.read() self.font = core.getfont( - "", size, index, encoding, self.font_bytes, layout_engine) + "", size, index, encoding, self.font_bytes, layout_engine + ) + + if isPath(font): + if sys.platform == "win32": + font_bytes_path = font if isinstance(font, bytes) else font.encode() + try: + font_bytes_path.decode("ascii") + except UnicodeDecodeError: + # FreeType cannot load fonts with non-ASCII characters on Windows + # So load it into memory first + with open(font, "rb") as f: + load_from_bytes(f) + return + self.font = core.getfont( + font, size, index, encoding, layout_engine=layout_engine + ) + else: + load_from_bytes(font) def _multiline_split(self, text): split_character = "\n" if isinstance(text, str) else b"\n" return text.split(split_character) def getname(self): + """ + :return: A tuple of the font family (e.g. Helvetica) and the font style + (e.g. Bold) + """ return self.font.family, self.font.style def getmetrics(self): + """ + :return: A tuple of the font ascent (the distance from the baseline to + the highest outline point) and descent (the distance from the + baseline to the lowest outline point, a negative value) + """ return self.font.ascent, self.font.descent - def getsize(self, text, direction=None, features=None): - size, offset = self.font.getsize(text, direction, features) - return (size[0] + offset[0], size[1] + offset[1]) + def getsize( + self, text, direction=None, features=None, language=None, stroke_width=0 + ): + """ + Returns width and height (in pixels) of given text if rendered in font with + provided direction, features, and language. - def getsize_multiline(self, text, direction=None, - spacing=4, features=None): + :param text: Text to measure. + + :param direction: Direction of the text. It can be 'rtl' (right to + left), 'ltr' (left to right) or 'ttb' (top to bottom). + Requires libraqm. + + .. versionadded:: 4.2.0 + + :param features: A list of OpenType font features to be used during text + layout. This is usually used to turn on optional + font features that are not enabled by default, + for example 'dlig' or 'ss01', but can be also + used to turn off default font features for + example '-liga' to disable ligatures or '-kern' + to disable kerning. To get all supported + features, see + https://docs.microsoft.com/en-us/typography/opentype/spec/featurelist + Requires libraqm. + + .. versionadded:: 4.2.0 + + :param language: Language of the text. Different languages may use + different glyph shapes or ligatures. This parameter tells + the font which language the text is in, and to apply the + correct substitutions as appropriate, if available. + It should be a `BCP 47 language code + ` + Requires libraqm. + + .. versionadded:: 6.0.0 + + :param stroke_width: The width of the text stroke. + + .. versionadded:: 6.2.0 + + :return: (width, height) + """ + size, offset = self.font.getsize(text, direction, features, language) + return ( + size[0] + stroke_width * 2 + offset[0], + size[1] + stroke_width * 2 + offset[1], + ) + + def getsize_multiline( + self, + text, + direction=None, + spacing=4, + features=None, + language=None, + stroke_width=0, + ): + """ + Returns width and height (in pixels) of given text if rendered in font + with provided direction, features, and language, while respecting + newline characters. + + :param text: Text to measure. + + :param direction: Direction of the text. It can be 'rtl' (right to + left), 'ltr' (left to right) or 'ttb' (top to bottom). + Requires libraqm. + + :param spacing: The vertical gap between lines, defaulting to 4 pixels. + + :param features: A list of OpenType font features to be used during text + layout. This is usually used to turn on optional + font features that are not enabled by default, + for example 'dlig' or 'ss01', but can be also + used to turn off default font features for + example '-liga' to disable ligatures or '-kern' + to disable kerning. To get all supported + features, see + https://docs.microsoft.com/en-us/typography/opentype/spec/featurelist + Requires libraqm. + + :param language: Language of the text. Different languages may use + different glyph shapes or ligatures. This parameter tells + the font which language the text is in, and to apply the + correct substitutions as appropriate, if available. + It should be a `BCP 47 language code + ` + Requires libraqm. + + .. versionadded:: 6.0.0 + + :param stroke_width: The width of the text stroke. + + .. versionadded:: 6.2.0 + + :return: (width, height) + """ max_width = 0 lines = self._multiline_split(text) - line_spacing = self.getsize('A')[1] + spacing + line_spacing = self.getsize("A", stroke_width=stroke_width)[1] + spacing for line in lines: - line_width, line_height = self.getsize(line, direction, features) + line_width, line_height = self.getsize( + line, direction, features, language, stroke_width + ) max_width = max(max_width, line_width) - return max_width, len(lines)*line_spacing - spacing + return max_width, len(lines) * line_spacing - spacing def getoffset(self, text): + """ + Returns the offset of given text. This is the gap between the + starting coordinate and the first marking. Note that this gap is + included in the result of :py:func:`~PIL.ImageFont.FreeTypeFont.getsize`. + + :param text: Text to measure. + + :return: A tuple of the x and y offset + """ return self.font.getsize(text)[1] - def getmask(self, text, mode="", direction=None, features=None): - return self.getmask2(text, mode, direction=direction, - features=features)[0] + def getmask( + self, + text, + mode="", + direction=None, + features=None, + language=None, + stroke_width=0, + ): + """ + Create a bitmap for the text. - def getmask2(self, text, mode="", fill=Image.core.fill, direction=None, - features=None, *args, **kwargs): - size, offset = self.font.getsize(text, direction, features) + If the font uses antialiasing, the bitmap should have mode ``L`` and use a + maximum value of 255. Otherwise, it should have mode ``1``. + + :param text: Text to render. + :param mode: Used by some graphics drivers to indicate what mode the + driver prefers; if empty, the renderer may return either + mode. Note that the mode is always a string, to simplify + C-level implementations. + + .. versionadded:: 1.1.5 + + :param direction: Direction of the text. It can be 'rtl' (right to + left), 'ltr' (left to right) or 'ttb' (top to bottom). + Requires libraqm. + + .. versionadded:: 4.2.0 + + :param features: A list of OpenType font features to be used during text + layout. This is usually used to turn on optional + font features that are not enabled by default, + for example 'dlig' or 'ss01', but can be also + used to turn off default font features for + example '-liga' to disable ligatures or '-kern' + to disable kerning. To get all supported + features, see + https://docs.microsoft.com/en-us/typography/opentype/spec/featurelist + Requires libraqm. + + .. versionadded:: 4.2.0 + + :param language: Language of the text. Different languages may use + different glyph shapes or ligatures. This parameter tells + the font which language the text is in, and to apply the + correct substitutions as appropriate, if available. + It should be a `BCP 47 language code + ` + Requires libraqm. + + .. versionadded:: 6.0.0 + + :param stroke_width: The width of the text stroke. + + .. versionadded:: 6.2.0 + + :return: An internal PIL storage memory instance as defined by the + :py:mod:`PIL.Image.core` interface module. + """ + return self.getmask2( + text, + mode, + direction=direction, + features=features, + language=language, + stroke_width=stroke_width, + )[0] + + def getmask2( + self, + text, + mode="", + fill=Image.core.fill, + direction=None, + features=None, + language=None, + stroke_width=0, + *args, + **kwargs + ): + """ + Create a bitmap for the text. + + If the font uses antialiasing, the bitmap should have mode ``L`` and use a + maximum value of 255. Otherwise, it should have mode ``1``. + + :param text: Text to render. + :param mode: Used by some graphics drivers to indicate what mode the + driver prefers; if empty, the renderer may return either + mode. Note that the mode is always a string, to simplify + C-level implementations. + + .. versionadded:: 1.1.5 + + :param direction: Direction of the text. It can be 'rtl' (right to + left), 'ltr' (left to right) or 'ttb' (top to bottom). + Requires libraqm. + + .. versionadded:: 4.2.0 + + :param features: A list of OpenType font features to be used during text + layout. This is usually used to turn on optional + font features that are not enabled by default, + for example 'dlig' or 'ss01', but can be also + used to turn off default font features for + example '-liga' to disable ligatures or '-kern' + to disable kerning. To get all supported + features, see + https://docs.microsoft.com/en-us/typography/opentype/spec/featurelist + Requires libraqm. + + .. versionadded:: 4.2.0 + + :param language: Language of the text. Different languages may use + different glyph shapes or ligatures. This parameter tells + the font which language the text is in, and to apply the + correct substitutions as appropriate, if available. + It should be a `BCP 47 language code + ` + Requires libraqm. + + .. versionadded:: 6.0.0 + + :param stroke_width: The width of the text stroke. + + .. versionadded:: 6.2.0 + + :return: A tuple of an internal PIL storage memory instance as defined by the + :py:mod:`PIL.Image.core` interface module, and the text offset, the + gap between the starting coordinate and the first marking + """ + size, offset = self.font.getsize(text, direction, features, language) + size = size[0] + stroke_width * 2, size[1] + stroke_width * 2 im = fill("L", size, 0) - self.font.render(text, im.id, mode == "1", direction, features) + self.font.render( + text, im.id, mode == "1", direction, features, language, stroke_width + ) return im, offset - def font_variant(self, font=None, size=None, index=None, encoding=None, - layout_engine=None): + def font_variant( + self, font=None, size=None, index=None, encoding=None, layout_engine=None + ): """ Create a copy of this FreeTypeFont object, using any specified arguments to override the settings. @@ -203,9 +485,62 @@ class FreeTypeFont(object): size=self.size if size is None else size, index=self.index if index is None else index, encoding=self.encoding if encoding is None else encoding, - layout_engine=self.layout_engine if layout_engine is None else layout_engine + layout_engine=layout_engine or self.layout_engine, ) + def get_variation_names(self): + """ + :returns: A list of the named styles in a variation font. + :exception IOError: If the font is not a variation font. + """ + try: + names = self.font.getvarnames() + except AttributeError: + raise NotImplementedError("FreeType 2.9.1 or greater is required") + return [name.replace(b"\x00", b"") for name in names] + + def set_variation_by_name(self, name): + """ + :param name: The name of the style. + :exception IOError: If the font is not a variation font. + """ + names = self.get_variation_names() + if not isinstance(name, bytes): + name = name.encode() + index = names.index(name) + + if index == getattr(self, "_last_variation_index", None): + # When the same name is set twice in a row, + # there is an 'unknown freetype error' + # https://savannah.nongnu.org/bugs/?56186 + return + self._last_variation_index = index + + self.font.setvarname(index) + + def get_variation_axes(self): + """ + :returns: A list of the axes in a variation font. + :exception IOError: If the font is not a variation font. + """ + try: + axes = self.font.getvaraxes() + except AttributeError: + raise NotImplementedError("FreeType 2.9.1 or greater is required") + for axis in axes: + axis["name"] = axis["name"].replace(b"\x00", b"") + return axes + + def set_variation_by_axes(self, axes): + """ + :param axes: A list of values for each axis. + :exception IOError: If the font is not a variation font. + """ + try: + self.font.setvaraxes(axes) + except AttributeError: + raise NotImplementedError("FreeType 2.9.1 or greater is required") + class TransposedFont(object): "Wrapper for writing rotated or mirrored text" @@ -250,35 +585,62 @@ def load(filename): return f -def truetype(font=None, size=10, index=0, encoding="", - layout_engine=None): +def truetype(font=None, size=10, index=0, encoding="", layout_engine=None): """ Load a TrueType or OpenType font from a file or file-like object, and create a font object. This function loads a font object from the given file or file-like object, and creates a font object for a font of the given size. + Pillow uses FreeType to open font files. If you are opening many fonts + simultaneously on Windows, be aware that Windows limits the number of files + that can be open in C at once to 512. If you approach that limit, an + ``OSError`` may be thrown, reporting that FreeType "cannot open resource". + This function requires the _imagingft service. :param font: A filename or file-like object containing a TrueType font. - Under Windows, if the file is not found in this filename, - the loader also looks in Windows :file:`fonts/` directory. + If the file is not found in this filename, the loader may also + search in other directories, such as the :file:`fonts/` + directory on Windows or :file:`/Library/Fonts/`, + :file:`/System/Library/Fonts/` and :file:`~/Library/Fonts/` on + macOS. + :param size: The requested size, in points. :param index: Which font face to load (default is first available face). - :param encoding: Which font encoding to use (default is Unicode). Common - encodings are "unic" (Unicode), "symb" (Microsoft - Symbol), "ADOB" (Adobe Standard), "ADBE" (Adobe Expert), - and "armn" (Apple Roman). See the FreeType documentation - for more information. + :param encoding: Which font encoding to use (default is Unicode). Possible + encodings include (see the FreeType documentation for more + information): + + * "unic" (Unicode) + * "symb" (Microsoft Symbol) + * "ADOB" (Adobe Standard) + * "ADBE" (Adobe Expert) + * "ADBC" (Adobe Custom) + * "armn" (Apple Roman) + * "sjis" (Shift JIS) + * "gb " (PRC) + * "big5" + * "wans" (Extended Wansung) + * "joha" (Johab) + * "lat1" (Latin-1) + + This specifies the character set to use. It does not alter the + encoding of any text provided in subsequent operations. :param layout_engine: Which layout engine to use, if available: `ImageFont.LAYOUT_BASIC` or `ImageFont.LAYOUT_RAQM`. :return: A font object. :exception IOError: If the file could not be read. """ - try: + def freetype(font): return FreeTypeFont(font, size, index, encoding, layout_engine) + + try: + return freetype(font) except IOError: + if not isPath(font): + raise ttf_filename = os.path.basename(font) dirs = [] @@ -289,17 +651,19 @@ def truetype(font=None, size=10, index=0, encoding="", windir = os.environ.get("WINDIR") if windir: dirs.append(os.path.join(windir, "fonts")) - elif sys.platform in ('linux', 'linux2'): + elif sys.platform in ("linux", "linux2"): lindirs = os.environ.get("XDG_DATA_DIRS", "") if not lindirs: # According to the freedesktop spec, XDG_DATA_DIRS should # default to /usr/share - lindirs = '/usr/share' - dirs += [os.path.join(lindir, "fonts") - for lindir in lindirs.split(":")] - elif sys.platform == 'darwin': - dirs += ['/Library/Fonts', '/System/Library/Fonts', - os.path.expanduser('~/Library/Fonts')] + lindirs = "/usr/share" + dirs += [os.path.join(lindir, "fonts") for lindir in lindirs.split(":")] + elif sys.platform == "darwin": + dirs += [ + "/Library/Fonts", + "/System/Library/Fonts", + os.path.expanduser("~/Library/Fonts"), + ] ext = os.path.splitext(ttf_filename)[1] first_font_with_a_different_extension = None @@ -307,21 +671,15 @@ def truetype(font=None, size=10, index=0, encoding="", for walkroot, walkdir, walkfilenames in os.walk(directory): for walkfilename in walkfilenames: if ext and walkfilename == ttf_filename: + return freetype(os.path.join(walkroot, walkfilename)) + elif not ext and os.path.splitext(walkfilename)[0] == ttf_filename: fontpath = os.path.join(walkroot, walkfilename) - return FreeTypeFont(fontpath, size, index, - encoding, layout_engine) - elif (not ext and - os.path.splitext(walkfilename)[0] == ttf_filename): - fontpath = os.path.join(walkroot, walkfilename) - if os.path.splitext(fontpath)[1] == '.ttf': - return FreeTypeFont(fontpath, size, index, - encoding, layout_engine) - if not ext \ - and first_font_with_a_different_extension is None: + if os.path.splitext(fontpath)[1] == ".ttf": + return freetype(fontpath) + if not ext and first_font_with_a_different_extension is None: first_font_with_a_different_extension = fontpath if first_font_with_a_different_extension: - return FreeTypeFont(first_font_with_a_different_extension, size, - index, encoding, layout_engine) + return freetype(first_font_with_a_different_extension) raise @@ -357,10 +715,13 @@ def load_default(): """ from io import BytesIO import base64 + f = ImageFont() f._load_pilfont_data( # courB08 - BytesIO(base64.b64decode(b''' + BytesIO( + base64.b64decode( + b""" UElMZm9udAo7Ozs7OzsxMDsKREFUQQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA @@ -452,7 +813,13 @@ AJsAEQAGAAAAAP/6AAX//wCbAAoAoAAPAAYAAAAA//oABQABAKAACgClABEABgAA////+AAGAAAA pQAKAKwAEgAGAAD////4AAYAAACsAAoAswASAAYAAP////gABgAAALMACgC6ABIABgAA////+QAG AAAAugAKAMEAEQAGAAD////4AAYAAgDBAAoAyAAUAAYAAP////kABQACAMgACgDOABMABgAA//// +QAGAAIAzgAKANUAEw== -''')), Image.open(BytesIO(base64.b64decode(b''' +""" + ) + ), + Image.open( + BytesIO( + base64.b64decode( + b""" iVBORw0KGgoAAAANSUhEUgAAAx4AAAAUAQAAAAArMtZoAAAEwElEQVR4nABlAJr/AHVE4czCI/4u Mc4b7vuds/xzjz5/3/7u/n9vMe7vnfH/9++vPn/xyf5zhxzjt8GHw8+2d83u8x27199/nxuQ6Od9 M43/5z2I+9n9ZtmDBwMQECDRQw/eQIQohJXxpBCNVE6QCCAAAAD//wBlAJr/AgALyj1t/wINwq0g @@ -476,5 +843,9 @@ evta/58PTEWzr21hufPjA8N+qlnBwAAAAAD//2JiWLci5v1+HmFXDqcnULE/MxgYGBj+f6CaJQAA AAD//2Ji2FrkY3iYpYC5qDeGgeEMAwPDvwQBBoYvcTwOVLMEAAAA//9isDBgkP///0EOg9z35v// Gc/eeW7BwPj5+QGZhANUswMAAAD//2JgqGBgYGBgqEMXlvhMPUsAAAAA//8iYDd1AAAAAP//AwDR w7IkEbzhVQAAAABJRU5ErkJggg== -''')))) +""" + ) + ) + ), + ) return f diff --git a/server/www/packages/packages-linux/x64/PIL/ImageGrab.py b/server/www/packages/packages-linux/x64/PIL/ImageGrab.py index 712b02c..9b44135 100644 --- a/server/www/packages/packages-linux/x64/PIL/ImageGrab.py +++ b/server/www/packages/packages-linux/x64/PIL/ImageGrab.py @@ -15,11 +15,9 @@ # See the README file for information on usage and redistribution. # -from . import Image - import sys -if sys.platform not in ["win32", "darwin"]: - raise ImportError("ImageGrab is macOS and Windows only") + +from . import Image if sys.platform == "win32": grabber = Image.core.grabscreen @@ -27,38 +25,51 @@ elif sys.platform == "darwin": import os import tempfile import subprocess +else: + raise ImportError("ImageGrab is macOS and Windows only") -def grab(bbox=None): +def grab(bbox=None, include_layered_windows=False, all_screens=False): if sys.platform == "darwin": - fh, filepath = tempfile.mkstemp('.png') + fh, filepath = tempfile.mkstemp(".png") os.close(fh) - subprocess.call(['screencapture', '-x', filepath]) + subprocess.call(["screencapture", "-x", filepath]) im = Image.open(filepath) im.load() os.unlink(filepath) + if bbox: + im = im.crop(bbox) else: - size, data = grabber() + offset, size, data = grabber(include_layered_windows, all_screens) im = Image.frombytes( - "RGB", size, data, + "RGB", + size, + data, # RGB, 32-bit line padding, origin lower left corner - "raw", "BGR", (size[0]*3 + 3) & -4, -1 - ) - if bbox: - im = im.crop(bbox) + "raw", + "BGR", + (size[0] * 3 + 3) & -4, + -1, + ) + if bbox: + x0, y0 = offset + left, top, right, bottom = bbox + im = im.crop((left - x0, top - y0, right - x0, bottom - y0)) return im def grabclipboard(): if sys.platform == "darwin": - fh, filepath = tempfile.mkstemp('.jpg') + fh, filepath = tempfile.mkstemp(".jpg") os.close(fh) commands = [ - "set theFile to (open for access POSIX file \""+filepath+"\" with write permission)", + 'set theFile to (open for access POSIX file "' + + filepath + + '" with write permission)', "try", - "write (the clipboard as JPEG picture) to theFile", + " write (the clipboard as JPEG picture) to theFile", "end try", - "close access theFile" + "close access theFile", ] script = ["osascript"] for command in commands: @@ -76,5 +87,6 @@ def grabclipboard(): if isinstance(data, bytes): from . import BmpImagePlugin import io + return BmpImagePlugin.DibImageFile(io.BytesIO(data)) return data diff --git a/server/www/packages/packages-linux/x64/PIL/ImageMath.py b/server/www/packages/packages-linux/x64/PIL/ImageMath.py index d985877..392151c 100644 --- a/server/www/packages/packages-linux/x64/PIL/ImageMath.py +++ b/server/www/packages/packages-linux/x64/PIL/ImageMath.py @@ -22,13 +22,14 @@ try: import builtins except ImportError: import __builtin__ + builtins = __builtin__ VERBOSE = 0 def _isconstant(v): - return isinstance(v, int) or isinstance(v, float) + return isinstance(v, (int, float)) class _Operand(object): @@ -61,7 +62,7 @@ class _Operand(object): out = Image.new(mode or im1.mode, im1.size, None) im1.load() try: - op = getattr(_imagingmath, op+"_"+im1.mode) + op = getattr(_imagingmath, op + "_" + im1.mode) except AttributeError: raise TypeError("bad operand type for '%s'" % op) _imagingmath.unop(op, out.im.id, im1.im.id) @@ -78,8 +79,7 @@ class _Operand(object): raise ValueError("mode mismatch") if im1.size != im2.size: # crop both arguments to a common size - size = (min(im1.size[0], im2.size[0]), - min(im1.size[1], im2.size[1])) + size = (min(im1.size[0], im2.size[0]), min(im1.size[1], im2.size[1])) if im1.size != size: im1 = im1.crop((0, 0) + size) if im2.size != size: @@ -90,7 +90,7 @@ class _Operand(object): im1.load() im2.load() try: - op = getattr(_imagingmath, op+"_"+im1.mode) + op = getattr(_imagingmath, op + "_" + im1.mode) except AttributeError: raise TypeError("bad operand type for '%s'" % op) _imagingmath.binop(op, out.im.id, im1.im.id, im2.im.id) diff --git a/server/www/packages/packages-linux/x64/PIL/ImageMode.py b/server/www/packages/packages-linux/x64/PIL/ImageMode.py index 2b3377a..596be7b 100644 --- a/server/www/packages/packages-linux/x64/PIL/ImageMode.py +++ b/server/www/packages/packages-linux/x64/PIL/ImageMode.py @@ -37,20 +37,28 @@ def getmode(mode): # initialize mode cache from . import Image + modes = {} # core modes for m, (basemode, basetype, bands) in Image._MODEINFO.items(): modes[m] = ModeDescriptor(m, bands, basemode, basetype) # extra experimental modes - modes["RGBa"] = ModeDescriptor("RGBa", - ("R", "G", "B", "a"), "RGB", "L") + modes["RGBa"] = ModeDescriptor("RGBa", ("R", "G", "B", "a"), "RGB", "L") modes["LA"] = ModeDescriptor("LA", ("L", "A"), "L", "L") modes["La"] = ModeDescriptor("La", ("L", "a"), "L", "L") modes["PA"] = ModeDescriptor("PA", ("P", "A"), "RGB", "L") # mapping modes - modes["I;16"] = ModeDescriptor("I;16", "I", "L", "L") - modes["I;16L"] = ModeDescriptor("I;16L", "I", "L", "L") - modes["I;16B"] = ModeDescriptor("I;16B", "I", "L", "L") + for i16mode in ( + "I;16", + "I;16S", + "I;16L", + "I;16LS", + "I;16B", + "I;16BS", + "I;16N", + "I;16NS", + ): + modes[i16mode] = ModeDescriptor(i16mode, ("I",), "L", "L") # set global mode cache atomically _modes = modes return _modes[mode] diff --git a/server/www/packages/packages-linux/x64/PIL/ImageMorph.py b/server/www/packages/packages-linux/x64/PIL/ImageMorph.py index 54ceb79..6119923 100644 --- a/server/www/packages/packages-linux/x64/PIL/ImageMorph.py +++ b/server/www/packages/packages-linux/x64/PIL/ImageMorph.py @@ -7,11 +7,25 @@ from __future__ import print_function -from . import Image, _imagingmorph import re +from . import Image, _imagingmorph + LUT_SIZE = 1 << 9 +# fmt: off +ROTATION_MATRIX = [ + 6, 3, 0, + 7, 4, 1, + 8, 5, 2, +] +MIRROR_MATRIX = [ + 2, 1, 0, + 5, 4, 3, + 8, 7, 6, +] +# fmt: on + class LutBuilder(object): """A class for building a MorphLut from a descriptive language @@ -48,6 +62,7 @@ class LutBuilder(object): lut = lb.build_lut() """ + def __init__(self, patterns=None, op_name=None): if patterns is not None: self.patterns = patterns @@ -56,20 +71,19 @@ class LutBuilder(object): self.lut = None if op_name is not None: known_patterns = { - 'corner': ['1:(... ... ...)->0', - '4:(00. 01. ...)->1'], - 'dilation4': ['4:(... .0. .1.)->1'], - 'dilation8': ['4:(... .0. .1.)->1', - '4:(... .0. ..1)->1'], - 'erosion4': ['4:(... .1. .0.)->0'], - 'erosion8': ['4:(... .1. .0.)->0', - '4:(... .1. ..0)->0'], - 'edge': ['1:(... ... ...)->0', - '4:(.0. .1. ...)->1', - '4:(01. .1. ...)->1'] + "corner": ["1:(... ... ...)->0", "4:(00. 01. ...)->1"], + "dilation4": ["4:(... .0. .1.)->1"], + "dilation8": ["4:(... .0. .1.)->1", "4:(... .0. ..1)->1"], + "erosion4": ["4:(... .1. .0.)->0"], + "erosion8": ["4:(... .1. .0.)->0", "4:(... .1. ..0)->0"], + "edge": [ + "1:(... ... ...)->0", + "4:(.0. .1. ...)->1", + "4:(01. .1. ...)->1", + ], } if op_name not in known_patterns: - raise Exception('Unknown pattern '+op_name+'!') + raise Exception("Unknown pattern " + op_name + "!") self.patterns = known_patterns[op_name] @@ -88,8 +102,8 @@ class LutBuilder(object): """string_permute takes a pattern and a permutation and returns the string permuted according to the permutation list. """ - assert(len(permutation) == 9) - return ''.join(pattern[p] for p in permutation) + assert len(permutation) == 9 + return "".join(pattern[p] for p in permutation) def _pattern_permute(self, basic_pattern, options, basic_result): """pattern_permute takes a basic pattern and its result and clones @@ -98,32 +112,25 @@ class LutBuilder(object): patterns = [(basic_pattern, basic_result)] # rotations - if '4' in options: + if "4" in options: res = patterns[-1][1] for i in range(4): patterns.append( - (self._string_permute(patterns[-1][0], [6, 3, 0, - 7, 4, 1, - 8, 5, 2]), res)) + (self._string_permute(patterns[-1][0], ROTATION_MATRIX), res) + ) # mirror - if 'M' in options: + if "M" in options: n = len(patterns) for pattern, res in patterns[0:n]: - patterns.append( - (self._string_permute(pattern, [2, 1, 0, - 5, 4, 3, - 8, 7, 6]), res)) + patterns.append((self._string_permute(pattern, MIRROR_MATRIX), res)) # negate - if 'N' in options: + if "N" in options: n = len(patterns) for pattern, res in patterns[0:n]: # Swap 0 and 1 - pattern = (pattern - .replace('0', 'Z') - .replace('1', '0') - .replace('Z', '1')) - res = 1-int(res) + pattern = pattern.replace("0", "Z").replace("1", "0").replace("Z", "1") + res = 1 - int(res) patterns.append((pattern, res)) return patterns @@ -138,22 +145,21 @@ class LutBuilder(object): # Parse and create symmetries of the patterns strings for p in self.patterns: - m = re.search( - r'(\w*):?\s*\((.+?)\)\s*->\s*(\d)', p.replace('\n', '')) + m = re.search(r"(\w*):?\s*\((.+?)\)\s*->\s*(\d)", p.replace("\n", "")) if not m: - raise Exception('Syntax error in pattern "'+p+'"') + raise Exception('Syntax error in pattern "' + p + '"') options = m.group(1) pattern = m.group(2) result = int(m.group(3)) # Get rid of spaces - pattern = pattern.replace(' ', '').replace('\n', '') + pattern = pattern.replace(" ", "").replace("\n", "") patterns += self._pattern_permute(pattern, options, result) # compile the patterns into regular expressions for speed for i, pattern in enumerate(patterns): - p = pattern[0].replace('.', 'X').replace('X', '[01]') + p = pattern[0].replace(".", "X").replace("X", "[01]") p = re.compile(p) patterns[i] = (p, pattern[1]) @@ -163,7 +169,7 @@ class LutBuilder(object): for i in range(LUT_SIZE): # Build the bit pattern bitpattern = bin(i)[2:] - bitpattern = ('0'*(9-len(bitpattern)) + bitpattern)[::-1] + bitpattern = ("0" * (9 - len(bitpattern)) + bitpattern)[::-1] for p, r in patterns: if p.match(bitpattern): @@ -175,10 +181,7 @@ class LutBuilder(object): class MorphOp(object): """A class for binary morphological operators""" - def __init__(self, - lut=None, - op_name=None, - patterns=None): + def __init__(self, lut=None, op_name=None, patterns=None): """Create a binary morphological operator""" self.lut = lut if op_name is not None: @@ -192,13 +195,12 @@ class MorphOp(object): Returns a tuple of the number of changed pixels and the morphed image""" if self.lut is None: - raise Exception('No operator loaded') + raise Exception("No operator loaded") - if image.mode != 'L': - raise Exception('Image must be binary, meaning it must use mode L') + if image.mode != "L": + raise Exception("Image must be binary, meaning it must use mode L") outimage = Image.new(image.mode, image.size, None) - count = _imagingmorph.apply( - bytes(self.lut), image.im.id, outimage.im.id) + count = _imagingmorph.apply(bytes(self.lut), image.im.id, outimage.im.id) return count, outimage def match(self, image): @@ -208,10 +210,10 @@ class MorphOp(object): Returns a list of tuples of (x,y) coordinates of all matching pixels. See :ref:`coordinate-system`.""" if self.lut is None: - raise Exception('No operator loaded') + raise Exception("No operator loaded") - if image.mode != 'L': - raise Exception('Image must be binary, meaning it must use mode L') + if image.mode != "L": + raise Exception("Image must be binary, meaning it must use mode L") return _imagingmorph.match(bytes(self.lut), image.im.id) def get_on_pixels(self, image): @@ -220,24 +222,24 @@ class MorphOp(object): Returns a list of tuples of (x,y) coordinates of all matching pixels. See :ref:`coordinate-system`.""" - if image.mode != 'L': - raise Exception('Image must be binary, meaning it must use mode L') + if image.mode != "L": + raise Exception("Image must be binary, meaning it must use mode L") return _imagingmorph.get_on_pixels(image.im.id) def load_lut(self, filename): """Load an operator from an mrl file""" - with open(filename, 'rb') as f: + with open(filename, "rb") as f: self.lut = bytearray(f.read()) if len(self.lut) != LUT_SIZE: self.lut = None - raise Exception('Wrong size operator file!') + raise Exception("Wrong size operator file!") def save_lut(self, filename): """Save an operator to an mrl file""" if self.lut is None: - raise Exception('No operator loaded') - with open(filename, 'wb') as f: + raise Exception("No operator loaded") + with open(filename, "wb") as f: f.write(self.lut) def set_lut(self, lut): diff --git a/server/www/packages/packages-linux/x64/PIL/ImageOps.py b/server/www/packages/packages-linux/x64/PIL/ImageOps.py index 9f516ba..5052cb7 100644 --- a/server/www/packages/packages-linux/x64/PIL/ImageOps.py +++ b/server/www/packages/packages-linux/x64/PIL/ImageOps.py @@ -17,16 +17,16 @@ # See the README file for information on usage and redistribution. # +import functools +import operator + from . import Image from ._util import isStringType -import operator -import functools -import warnings - # # helpers + def _border(border): if isinstance(border, tuple): if len(border) == 2: @@ -41,6 +41,7 @@ def _border(border): def _color(color, mode): if isStringType(color): from . import ImageColor + color = ImageColor.getcolor(color, mode) return color @@ -56,6 +57,7 @@ def _lut(image, lut): else: raise IOError("not supported for this image mode") + # # actions @@ -76,7 +78,7 @@ def autocontrast(image, cutoff=0, ignore=None): histogram = image.histogram() lut = [] for layer in range(0, len(histogram), 256): - h = histogram[layer:layer+256] + h = histogram[layer : layer + 256] if ignore is not None: # get rid of outliers try: @@ -136,8 +138,7 @@ def autocontrast(image, cutoff=0, ignore=None): return _lut(image, lut) -def colorize(image, black, white, mid=None, blackpoint=0, - whitepoint=255, midpoint=127): +def colorize(image, black, white, mid=None, blackpoint=0, whitepoint=255, midpoint=127): """ Colorize grayscale image. This function calculates a color wedge which maps all black pixels in @@ -277,9 +278,7 @@ def crop(image, border=0): :return: An image. """ left, top, right, bottom = _border(border) - return image.crop( - (left, top, image.size[0]-right, image.size[1]-bottom) - ) + return image.crop((left, top, image.size[0] - right, image.size[1] - bottom)) def scale(image, factor, resample=Image.NEAREST): @@ -299,8 +298,7 @@ def scale(image, factor, resample=Image.NEAREST): elif factor <= 0: raise ValueError("the factor must be greater than 0") else: - size = (int(round(factor * image.width)), - int(round(factor * image.height))) + size = (int(round(factor * image.width)), int(round(factor * image.height))) return image.resize(size, resample) @@ -315,9 +313,7 @@ def deform(image, deformer, resample=Image.BILINEAR): in the PIL.Image.transform function. :return: An image. """ - return image.transform( - image.size, Image.MESH, deformer.getmesh(image), resample - ) + return image.transform(image.size, Image.MESH, deformer.getmesh(image), resample) def equalize(image, mask=None): @@ -336,7 +332,7 @@ def equalize(image, mask=None): h = image.histogram(mask) lut = [] for b in range(0, len(h), 256): - histo = [_f for _f in h[b:b+256] if _f] + histo = [_f for _f in h[b : b + 256] if _f] if len(histo) <= 1: lut.extend(list(range(256))) else: @@ -347,7 +343,7 @@ def equalize(image, mask=None): n = step // 2 for i in range(256): lut.append(n // step) - n = n + h[i+b] + n = n + h[i + b] return _lut(image, lut) @@ -380,9 +376,10 @@ def fit(image, size, method=Image.NEAREST, bleed=0.0, centering=(0.5, 0.5)): (width, height) tuple. :param method: What resampling method to use. Default is :py:attr:`PIL.Image.NEAREST`. - :param bleed: Remove a border around the outside of the image (from all + :param bleed: Remove a border around the outside of the image from all four edges. The value is a decimal percentage (use 0.01 for one percent). The default value is 0 (no border). + Cannot be greater than or equal to 0.5. :param centering: Control the cropping position. Use (0.5, 0.5) for center cropping (e.g. if cropping the width, take 50% off of the left side, and therefore 50% off the right side). @@ -400,66 +397,56 @@ def fit(image, size, method=Image.NEAREST, bleed=0.0, centering=(0.5, 0.5)): # kevin@cazabon.com # http://www.cazabon.com - # ensure inputs are valid - if not isinstance(centering, list): - centering = [centering[0], centering[1]] + # ensure centering is mutable + centering = list(centering) - if centering[0] > 1.0 or centering[0] < 0.0: - centering[0] = 0.50 - if centering[1] > 1.0 or centering[1] < 0.0: - centering[1] = 0.50 + if not 0.0 <= centering[0] <= 1.0: + centering[0] = 0.5 + if not 0.0 <= centering[1] <= 1.0: + centering[1] = 0.5 - if bleed > 0.49999 or bleed < 0.0: + if not 0.0 <= bleed < 0.5: bleed = 0.0 # calculate the area to use for resizing and cropping, subtracting # the 'bleed' around the edges # number of pixels to trim off on Top and Bottom, Left and Right - bleedPixels = ( - int((float(bleed) * float(image.size[0])) + 0.5), - int((float(bleed) * float(image.size[1])) + 0.5) - ) + bleed_pixels = (bleed * image.size[0], bleed * image.size[1]) - liveArea = (0, 0, image.size[0], image.size[1]) - if bleed > 0.0: - liveArea = ( - bleedPixels[0], bleedPixels[1], image.size[0] - bleedPixels[0] - 1, - image.size[1] - bleedPixels[1] - 1 - ) + live_size = ( + image.size[0] - bleed_pixels[0] * 2, + image.size[1] - bleed_pixels[1] * 2, + ) - liveSize = (liveArea[2] - liveArea[0], liveArea[3] - liveArea[1]) - - # calculate the aspect ratio of the liveArea - liveAreaAspectRatio = float(liveSize[0])/float(liveSize[1]) + # calculate the aspect ratio of the live_size + live_size_ratio = float(live_size[0]) / live_size[1] # calculate the aspect ratio of the output image - aspectRatio = float(size[0]) / float(size[1]) + output_ratio = float(size[0]) / size[1] # figure out if the sides or top/bottom will be cropped off - if liveAreaAspectRatio >= aspectRatio: - # liveArea is wider than what's needed, crop the sides - cropWidth = int((aspectRatio * float(liveSize[1])) + 0.5) - cropHeight = liveSize[1] + if live_size_ratio == output_ratio: + # live_size is already the needed ratio + crop_width = live_size[0] + crop_height = live_size[1] + elif live_size_ratio >= output_ratio: + # live_size is wider than what's needed, crop the sides + crop_width = output_ratio * live_size[1] + crop_height = live_size[1] else: - # liveArea is taller than what's needed, crop the top and bottom - cropWidth = liveSize[0] - cropHeight = int((float(liveSize[0])/aspectRatio) + 0.5) + # live_size is taller than what's needed, crop the top and bottom + crop_width = live_size[0] + crop_height = live_size[0] / output_ratio # make the crop - leftSide = int(liveArea[0] + (float(liveSize[0]-cropWidth) * centering[0])) - if leftSide < 0: - leftSide = 0 - topSide = int(liveArea[1] + (float(liveSize[1]-cropHeight) * centering[1])) - if topSide < 0: - topSide = 0 + crop_left = bleed_pixels[0] + (live_size[0] - crop_width) * centering[0] + crop_top = bleed_pixels[1] + (live_size[1] - crop_height) * centering[1] - out = image.crop( - (leftSide, topSide, leftSide + cropWidth, topSide + cropHeight) - ) + crop = (crop_left, crop_top, crop_left + crop_width, crop_top + crop_height) # resize the image and return it - return out.resize(size, method) + return image.resize(size, method, box=crop) def flip(image): @@ -491,7 +478,7 @@ def invert(image): """ lut = [] for i in range(256): - lut.append(255-i) + lut.append(255 - i) return _lut(image, lut) @@ -514,7 +501,7 @@ def posterize(image, bits): :return: An image. """ lut = [] - mask = ~(2**(8-bits)-1) + mask = ~(2 ** (8 - bits) - 1) for i in range(256): lut.append(i & mask) return _lut(image, lut) @@ -533,100 +520,32 @@ def solarize(image, threshold=128): if i < threshold: lut.append(i) else: - lut.append(255-i) + lut.append(255 - i) return _lut(image, lut) -# -------------------------------------------------------------------- -# PIL USM components, from Kevin Cazabon. - -def gaussian_blur(im, radius=None): - """ PIL_usm.gblur(im, [radius])""" - - warnings.warn( - 'PIL.ImageOps.gaussian_blur is deprecated. ' - 'Use PIL.ImageFilter.GaussianBlur instead. ' - 'This function will be removed in a future version.', - DeprecationWarning - ) - - if radius is None: - radius = 5.0 - - im.load() - - return im.im.gaussian_blur(radius) - - -def gblur(im, radius=None): - """ PIL_usm.gblur(im, [radius])""" - - warnings.warn( - 'PIL.ImageOps.gblur is deprecated. ' - 'Use PIL.ImageFilter.GaussianBlur instead. ' - 'This function will be removed in a future version.', - DeprecationWarning - ) - - return gaussian_blur(im, radius) - - -def unsharp_mask(im, radius=None, percent=None, threshold=None): - """ PIL_usm.usm(im, [radius, percent, threshold])""" - - warnings.warn( - 'PIL.ImageOps.unsharp_mask is deprecated. ' - 'Use PIL.ImageFilter.UnsharpMask instead. ' - 'This function will be removed in a future version.', - DeprecationWarning - ) - - if radius is None: - radius = 5.0 - if percent is None: - percent = 150 - if threshold is None: - threshold = 3 - - im.load() - - return im.im.unsharp_mask(radius, percent, threshold) - - -def usm(im, radius=None, percent=None, threshold=None): - """ PIL_usm.usm(im, [radius, percent, threshold])""" - - warnings.warn( - 'PIL.ImageOps.usm is deprecated. ' - 'Use PIL.ImageFilter.UnsharpMask instead. ' - 'This function will be removed in a future version.', - DeprecationWarning - ) - - return unsharp_mask(im, radius, percent, threshold) - - -def box_blur(image, radius): +def exif_transpose(image): """ - Blur the image by setting each pixel to the average value of the pixels - in a square box extending radius pixels in each direction. - Supports float radius of arbitrary size. Uses an optimized implementation - which runs in linear time relative to the size of the image - for any radius value. + If an image has an EXIF Orientation tag, return a new image that is + transposed accordingly. Otherwise, return a copy of the image. - :param image: The image to blur. - :param radius: Size of the box in one direction. Radius 0 does not blur, - returns an identical image. Radius 1 takes 1 pixel - in each direction, i.e. 9 pixels in total. + :param image: The image to transpose. :return: An image. """ - warnings.warn( - 'PIL.ImageOps.box_blur is deprecated. ' - 'Use PIL.ImageFilter.BoxBlur instead. ' - 'This function will be removed in a future version.', - DeprecationWarning - ) - - image.load() - - return image._new(image.im.box_blur(radius)) + exif = image.getexif() + orientation = exif.get(0x0112) + method = { + 2: Image.FLIP_LEFT_RIGHT, + 3: Image.ROTATE_180, + 4: Image.FLIP_TOP_BOTTOM, + 5: Image.TRANSPOSE, + 6: Image.ROTATE_270, + 7: Image.TRANSVERSE, + 8: Image.ROTATE_90, + }.get(orientation) + if method is not None: + transposed_image = image.transpose(method) + del exif[0x0112] + transposed_image.info["exif"] = exif.tobytes() + return transposed_image + return image.copy() diff --git a/server/www/packages/packages-linux/x64/PIL/ImagePalette.py b/server/www/packages/packages-linux/x64/PIL/ImagePalette.py index 81e99ab..2d4f5cb 100644 --- a/server/www/packages/packages-linux/x64/PIL/ImagePalette.py +++ b/server/www/packages/packages-linux/x64/PIL/ImagePalette.py @@ -17,7 +17,8 @@ # import array -from . import ImageColor, GimpPaletteFile, GimpGradientFile, PaletteFile + +from . import GimpGradientFile, GimpPaletteFile, ImageColor, PaletteFile class ImagePalette(object): @@ -38,11 +39,12 @@ class ImagePalette(object): def __init__(self, mode="RGB", palette=None, size=0): self.mode = mode self.rawmode = None # if set, palette contains raw data - self.palette = palette or bytearray(range(256))*len(self.mode) + self.palette = palette or bytearray(range(256)) * len(self.mode) self.colors = {} self.dirty = None - if ((size == 0 and len(self.mode)*256 != len(self.palette)) or - (size != 0 and size != len(self.palette))): + if (size == 0 and len(self.mode) * 256 != len(self.palette)) or ( + size != 0 and size != len(self.palette) + ): raise ValueError("wrong palette size") def copy(self): @@ -78,7 +80,7 @@ class ImagePalette(object): if isinstance(self.palette, bytes): return self.palette arr = array.array("B", self.palette) - if hasattr(arr, 'tobytes'): + if hasattr(arr, "tobytes"): return arr.tobytes() return arr.tostring() @@ -104,8 +106,8 @@ class ImagePalette(object): raise ValueError("cannot allocate more than 256 colors") self.colors[color] = index self.palette[index] = color[0] - self.palette[index+256] = color[1] - self.palette[index+512] = color[2] + self.palette[index + 256] = color[1] + self.palette[index + 512] = color[2] self.dirty = 1 return index else: @@ -124,7 +126,7 @@ class ImagePalette(object): fp.write("# Mode: %s\n" % self.mode) for i in range(256): fp.write("%d" % i) - for j in range(i*len(self.mode), (i+1)*len(self.mode)): + for j in range(i * len(self.mode), (i + 1) * len(self.mode)): try: fp.write(" %d" % self.palette[j]) except IndexError: @@ -136,6 +138,7 @@ class ImagePalette(object): # -------------------------------------------------------------------- # Internal + def raw(rawmode, data): palette = ImagePalette() palette.rawmode = rawmode @@ -147,11 +150,12 @@ def raw(rawmode, data): # -------------------------------------------------------------------- # Factories + def make_linear_lut(black, white): lut = [] if black == 0: for i in range(256): - lut.append(white*i//255) + lut.append(white * i // 255) else: raise NotImplementedError # FIXME return lut @@ -172,8 +176,9 @@ def negative(mode="RGB"): def random(mode="RGB"): from random import randint + palette = [] - for i in range(256*len(mode)): + for i in range(256 * len(mode)): palette.append(randint(0, 255)) return ImagePalette(mode, palette) @@ -199,7 +204,7 @@ def load(filename): for paletteHandler in [ GimpPaletteFile.GimpPaletteFile, GimpGradientFile.GimpGradientFile, - PaletteFile.PaletteFile + PaletteFile.PaletteFile, ]: try: fp.seek(0) diff --git a/server/www/packages/packages-linux/x64/PIL/ImagePath.py b/server/www/packages/packages-linux/x64/PIL/ImagePath.py index 8cbfec0..3d3538c 100644 --- a/server/www/packages/packages-linux/x64/PIL/ImagePath.py +++ b/server/www/packages/packages-linux/x64/PIL/ImagePath.py @@ -16,5 +16,4 @@ from . import Image - Path = Image.core.path diff --git a/server/www/packages/packages-linux/x64/PIL/ImageQt.py b/server/www/packages/packages-linux/x64/PIL/ImageQt.py index e602613..2edb0a1 100644 --- a/server/www/packages/packages-linux/x64/PIL/ImageQt.py +++ b/server/www/packages/packages-linux/x64/PIL/ImageQt.py @@ -16,34 +16,40 @@ # See the README file for information on usage and redistribution. # +import sys +import warnings +from io import BytesIO + from . import Image from ._util import isPath, py3 -from io import BytesIO -import sys -qt_versions = [ - ['5', 'PyQt5'], - ['side2', 'PySide2'], - ['4', 'PyQt4'], - ['side', 'PySide'] -] +qt_versions = [["5", "PyQt5"], ["side2", "PySide2"], ["4", "PyQt4"], ["side", "PySide"]] + +WARNING_TEXT = ( + "Support for EOL {} is deprecated and will be removed in a future version. " + "Please upgrade to PyQt5 or PySide2." +) + # If a version has already been imported, attempt it first -qt_versions.sort(key=lambda qt_version: qt_version[1] in sys.modules, - reverse=True) +qt_versions.sort(key=lambda qt_version: qt_version[1] in sys.modules, reverse=True) for qt_version, qt_module in qt_versions: try: - if qt_module == 'PyQt5': + if qt_module == "PyQt5": from PyQt5.QtGui import QImage, qRgba, QPixmap from PyQt5.QtCore import QBuffer, QIODevice - elif qt_module == 'PySide2': + elif qt_module == "PySide2": from PySide2.QtGui import QImage, qRgba, QPixmap from PySide2.QtCore import QBuffer, QIODevice - elif qt_module == 'PyQt4': + elif qt_module == "PyQt4": from PyQt4.QtGui import QImage, qRgba, QPixmap from PyQt4.QtCore import QBuffer, QIODevice - elif qt_module == 'PySide': + + warnings.warn(WARNING_TEXT.format(qt_module), DeprecationWarning) + elif qt_module == "PySide": from PySide.QtGui import QImage, qRgba, QPixmap from PySide.QtCore import QBuffer, QIODevice + + warnings.warn(WARNING_TEXT.format(qt_module), DeprecationWarning) except (ImportError, RuntimeError): continue qt_is_installed = True @@ -57,7 +63,7 @@ def rgb(r, g, b, a=255): """(Internal) Turns an RGB color into a Qt compatible color integer.""" # use qRgb to pack the colors, and then turn the resulting long # into a negative integer with the same bitpattern. - return (qRgba(r, g, b, a) & 0xffffffff) + return qRgba(r, g, b, a) & 0xFFFFFFFF def fromqimage(im): @@ -67,12 +73,12 @@ def fromqimage(im): """ buffer = QBuffer() buffer.open(QIODevice.ReadWrite) - # preserve alha channel with png + # preserve alpha channel with png # otherwise ppm is more friendly with Image.open if im.hasAlphaChannel(): - im.save(buffer, 'png') + im.save(buffer, "png") else: - im.save(buffer, 'ppm') + im.save(buffer, "ppm") b = BytesIO() try: @@ -105,11 +111,7 @@ def align8to32(bytes, width, mode): converts each scanline of data from 8 bit to 32 bit aligned """ - bits_per_pixel = { - '1': 1, - 'L': 8, - 'P': 8, - }[mode] + bits_per_pixel = {"1": 1, "L": 8, "P": 8}[mode] # calculate bytes per line and the extra padding if needed bits_per_line = bits_per_pixel * width @@ -124,10 +126,12 @@ def align8to32(bytes, width, mode): new_data = [] for i in range(len(bytes) // bytes_per_line): - new_data.append(bytes[i*bytes_per_line:(i+1)*bytes_per_line] - + b'\x00' * extra_padding) + new_data.append( + bytes[i * bytes_per_line : (i + 1) * bytes_per_line] + + b"\x00" * extra_padding + ) - return b''.join(new_data) + return b"".join(new_data) def _toqclass_helper(im): @@ -140,7 +144,7 @@ def _toqclass_helper(im): if py3: im = str(im.toUtf8(), "utf-8") else: - im = unicode(im.toUtf8(), "utf-8") + im = unicode(im.toUtf8(), "utf-8") # noqa: F821 if isPath(im): im = Image.open(im) @@ -156,7 +160,7 @@ def _toqclass_helper(im): colortable = [] palette = im.getpalette() for i in range(0, len(palette), 3): - colortable.append(rgb(*palette[i:i+3])) + colortable.append(rgb(*palette[i : i + 3])) elif im.mode == "RGB": data = im.tobytes("raw", "BGRX") format = QImage.Format_RGB32 @@ -172,33 +176,35 @@ def _toqclass_helper(im): raise ValueError("unsupported image mode %r" % im.mode) __data = data or align8to32(im.tobytes(), im.size[0], im.mode) - return { - 'data': __data, 'im': im, 'format': format, 'colortable': colortable - } + return {"data": __data, "im": im, "format": format, "colortable": colortable} if qt_is_installed: - class ImageQt(QImage): + class ImageQt(QImage): def __init__(self, im): """ An PIL image wrapper for Qt. This is a subclass of PyQt's QImage class. - :param im: A PIL Image object, or a file name (given either as Python - string or a PyQt string object). + :param im: A PIL Image object, or a file name (given either as + Python string or a PyQt string object). """ im_data = _toqclass_helper(im) # must keep a reference, or Qt will crash! # All QImage constructors that take data operate on an existing # buffer, so this buffer has to hang on for the life of the image. # Fixes https://github.com/python-pillow/Pillow/issues/1370 - self.__data = im_data['data'] - QImage.__init__(self, - self.__data, im_data['im'].size[0], - im_data['im'].size[1], im_data['format']) - if im_data['colortable']: - self.setColorTable(im_data['colortable']) + self.__data = im_data["data"] + QImage.__init__( + self, + self.__data, + im_data["im"].size[0], + im_data["im"].size[1], + im_data["format"], + ) + if im_data["colortable"]: + self.setColorTable(im_data["colortable"]) def toqimage(im): @@ -211,8 +217,8 @@ def toqpixmap(im): # result = QPixmap(im_data['im'].size[0], im_data['im'].size[1]) # result.loadFromData(im_data['data']) # Fix some strange bug that causes - if im.mode == 'RGB': - im = im.convert('RGBA') + if im.mode == "RGB": + im = im.convert("RGBA") qimage = toqimage(im) return QPixmap.fromImage(qimage) diff --git a/server/www/packages/packages-linux/x64/PIL/ImageSequence.py b/server/www/packages/packages-linux/x64/PIL/ImageSequence.py index 1fc6e5d..f9be92d 100644 --- a/server/www/packages/packages-linux/x64/PIL/ImageSequence.py +++ b/server/www/packages/packages-linux/x64/PIL/ImageSequence.py @@ -32,7 +32,7 @@ class Iterator(object): if not hasattr(im, "seek"): raise AttributeError("im must have seek method") self.im = im - self.position = 0 + self.position = getattr(self.im, "_min_frame", 0) def __getitem__(self, ix): try: @@ -54,3 +54,25 @@ class Iterator(object): def next(self): return self.__next__() + + +def all_frames(im, func=None): + """ + Applies a given function to all frames in an image or a list of images. + The frames are returned as a list of separate images. + + :param im: An image, or a list of images. + :param func: The function to apply to all of the image frames. + :returns: A list of images. + """ + if not isinstance(im, list): + im = [im] + + ims = [] + for imSequence in im: + current = imSequence.tell() + + ims += [im_frame.copy() for im_frame in Iterator(imSequence)] + + imSequence.seek(current) + return [func(im) for im in ims] if func else ims diff --git a/server/www/packages/packages-linux/x64/PIL/ImageShow.py b/server/www/packages/packages-linux/x64/PIL/ImageShow.py index b50d613..ca622c5 100644 --- a/server/www/packages/packages-linux/x64/PIL/ImageShow.py +++ b/server/www/packages/packages-linux/x64/PIL/ImageShow.py @@ -14,9 +14,12 @@ from __future__ import print_function -from PIL import Image import os +import subprocess import sys +import tempfile + +from PIL import Image if sys.version_info.major >= 3: from shlex import quote @@ -61,16 +64,12 @@ class Viewer(object): def show(self, image, **options): # save temporary image to disk - if image.mode[:4] == "I;16": - # @PIL88 @PIL101 - # "I;16" isn't an 'official' mode, but we still want to - # provide a simple way to show 16-bit images. - base = "L" - # FIXME: auto-contrast if max() > 255? - else: + if not ( + image.mode in ("1", "RGBA") or (self.format == "PNG" and image.mode == "LA") + ): base = Image.getmodebase(image.mode) - if base != image.mode and image.mode != "1" and image.mode != "RGBA": - image = image.convert(base) + if image.mode != base: + image = image.convert(base) return self.show_image(image, **options) @@ -99,18 +98,22 @@ class Viewer(object): os.system(self.get_command(file, **options)) return 1 + # -------------------------------------------------------------------- if sys.platform == "win32": class WindowsViewer(Viewer): - format = "BMP" + format = "PNG" + options = {"compress_level": 1} def get_command(self, file, **options): - return ('start "Pillow" /WAIT "%s" ' - '&& ping -n 2 127.0.0.1 >NUL ' - '&& del /f "%s"' % (file, file)) + return ( + 'start "Pillow" /WAIT "%s" ' + "&& ping -n 2 127.0.0.1 >NUL " + '&& del /f "%s"' % (file, file) + ) register(WindowsViewer) @@ -118,16 +121,33 @@ elif sys.platform == "darwin": class MacViewer(Viewer): format = "PNG" - options = {'compress_level': 1} + options = {"compress_level": 1} def get_command(self, file, **options): # on darwin open returns immediately resulting in the temp # file removal while app is opening - command = "open -a /Applications/Preview.app" - command = "(%s %s; sleep 20; rm -f %s)&" % (command, quote(file), - quote(file)) + command = "open -a Preview.app" + command = "(%s %s; sleep 20; rm -f %s)&" % ( + command, + quote(file), + quote(file), + ) return command + def show_file(self, file, **options): + """Display given file""" + fd, path = tempfile.mkstemp() + with os.fdopen(fd, "w") as f: + f.write(file) + with open(path, "r") as f: + subprocess.Popen( + ["im=$(cat); open -a Preview.app $im; sleep 20; rm -f $im"], + shell=True, + stdin=f, + ) + os.remove(path) + return 1 + register(MacViewer) else: @@ -146,13 +166,23 @@ else: class UnixViewer(Viewer): format = "PNG" - options = {'compress_level': 1} + options = {"compress_level": 1} + + def get_command(self, file, **options): + command = self.get_command_ex(file, **options)[0] + return "(%s %s; rm -f %s)&" % (command, quote(file), quote(file)) def show_file(self, file, **options): - command, executable = self.get_command_ex(file, **options) - command = "(%s %s; rm -f %s)&" % (command, quote(file), - quote(file)) - os.system(command) + """Display given file""" + fd, path = tempfile.mkstemp() + with os.fdopen(fd, "w") as f: + f.write(file) + with open(path, "r") as f: + command = self.get_command_ex(file, **options)[0] + subprocess.Popen( + ["im=$(cat);" + command + " $im; rm -f $im"], shell=True, stdin=f + ) + os.remove(path) return 1 # implementations diff --git a/server/www/packages/packages-linux/x64/PIL/ImageStat.py b/server/www/packages/packages-linux/x64/PIL/ImageStat.py index d4b38d8..9ba16fd 100644 --- a/server/www/packages/packages-linux/x64/PIL/ImageStat.py +++ b/server/www/packages/packages-linux/x64/PIL/ImageStat.py @@ -21,13 +21,12 @@ # See the README file for information on usage and redistribution. # +import functools import math import operator -import functools class Stat(object): - def __init__(self, image_or_list, mask=None): try: if mask: @@ -41,7 +40,7 @@ class Stat(object): self.bands = list(range(len(self.h) // 256)) def __getattr__(self, id): - "Calculate missing attribute" + """Calculate missing attribute""" if id[:4] == "_get": raise AttributeError(id) # calculate missing attribute @@ -50,7 +49,7 @@ class Stat(object): return v def _getextrema(self): - "Get min/max values for each band in the image" + """Get min/max values for each band in the image""" def minmax(histogram): n = 255 @@ -67,15 +66,15 @@ class Stat(object): return v def _getcount(self): - "Get total number of pixels in each layer" + """Get total number of pixels in each layer""" v = [] for i in range(0, len(self.h), 256): - v.append(functools.reduce(operator.add, self.h[i:i+256])) + v.append(functools.reduce(operator.add, self.h[i : i + 256])) return v def _getsum(self): - "Get sum of all pixels in each layer" + """Get sum of all pixels in each layer""" v = [] for i in range(0, len(self.h), 256): @@ -86,7 +85,7 @@ class Stat(object): return v def _getsum2(self): - "Get squared sum of all pixels in each layer" + """Get squared sum of all pixels in each layer""" v = [] for i in range(0, len(self.h), 256): @@ -97,7 +96,7 @@ class Stat(object): return v def _getmean(self): - "Get average pixel level for each layer" + """Get average pixel level for each layer""" v = [] for i in self.bands: @@ -105,22 +104,22 @@ class Stat(object): return v def _getmedian(self): - "Get median pixel level for each layer" + """Get median pixel level for each layer""" v = [] for i in self.bands: s = 0 - half = self.count[i]//2 + half = self.count[i] // 2 b = i * 256 for j in range(256): - s = s + self.h[b+j] + s = s + self.h[b + j] if s > half: break v.append(j) return v def _getrms(self): - "Get RMS for each layer" + """Get RMS for each layer""" v = [] for i in self.bands: @@ -128,16 +127,16 @@ class Stat(object): return v def _getvar(self): - "Get variance for each layer" + """Get variance for each layer""" v = [] for i in self.bands: n = self.count[i] - v.append((self.sum2[i]-(self.sum[i]**2.0)/n)/n) + v.append((self.sum2[i] - (self.sum[i] ** 2.0) / n) / n) return v def _getstddev(self): - "Get standard deviation for each layer" + """Get standard deviation for each layer""" v = [] for i in self.bands: diff --git a/server/www/packages/packages-linux/x64/PIL/ImageTk.py b/server/www/packages/packages-linux/x64/PIL/ImageTk.py index c56f556..fd48000 100644 --- a/server/www/packages/packages-linux/x64/PIL/ImageTk.py +++ b/server/www/packages/packages-linux/x64/PIL/ImageTk.py @@ -26,15 +26,15 @@ # import sys +from io import BytesIO + +from . import Image if sys.version_info.major > 2: import tkinter else: import Tkinter as tkinter -from . import Image -from io import BytesIO - # -------------------------------------------------------------------- # Check for Tkinter interface hooks @@ -67,6 +67,7 @@ def _get_image_from_kw(kw): # -------------------------------------------------------------------- # PhotoImage + class PhotoImage(object): """ A Tkinter-compatible photo image. This can be used @@ -124,7 +125,7 @@ class PhotoImage(object): self.__photo.name = None try: self.__photo.tk.call("image", "delete", name) - except: + except Exception: pass # ignore internal errors def __str__(self): @@ -183,17 +184,18 @@ class PhotoImage(object): # activate Tkinter hook try: from . import _imagingtk + try: - if hasattr(tk, 'interp'): + if hasattr(tk, "interp"): # Required for PyPy, which always has CFFI installed from cffi import FFI + ffi = FFI() # PyPy is using an FFI CDATA element # (Pdb) self.tk.interp # - _imagingtk.tkinit( - int(ffi.cast("uintptr_t", tk.interp)), 1) + _imagingtk.tkinit(int(ffi.cast("uintptr_t", tk.interp)), 1) else: _imagingtk.tkinit(tk.interpaddr(), 1) except AttributeError: @@ -202,6 +204,7 @@ class PhotoImage(object): except (ImportError, AttributeError, tkinter.TclError): raise # configuration problem; cannot attach to Tkinter + # -------------------------------------------------------------------- # BitmapImage @@ -244,7 +247,7 @@ class BitmapImage(object): self.__photo.name = None try: self.__photo.tk.call("image", "delete", name) - except: + except Exception: pass # ignore internal errors def width(self): @@ -275,10 +278,13 @@ class BitmapImage(object): def getimage(photo): - """ This function is unimplemented """ - """Copies the contents of a PhotoImage to a PIL image memory.""" - photo.tk.call("PyImagingPhotoGet", photo) + im = Image.new("RGBA", (photo.width(), photo.height())) + block = im.im + + photo.tk.call("PyImagingPhotoGet", photo, block.id) + + return im def _show(image, title): @@ -290,8 +296,7 @@ def _show(image, title): self.image = BitmapImage(im, foreground="white", master=master) else: self.image = PhotoImage(im, master=master) - tkinter.Label.__init__(self, master, image=self.image, - bg="black", bd=0) + tkinter.Label.__init__(self, master, image=self.image, bg="black", bd=0) if not tkinter._default_root: raise IOError("tkinter not initialized") diff --git a/server/www/packages/packages-linux/x64/PIL/ImageTransform.py b/server/www/packages/packages-linux/x64/PIL/ImageTransform.py index c3f6af8..77791ab 100644 --- a/server/www/packages/packages-linux/x64/PIL/ImageTransform.py +++ b/server/www/packages/packages-linux/x64/PIL/ImageTransform.py @@ -46,6 +46,7 @@ class AffineTransform(Transform): :param matrix: A 6-tuple (a, b, c, d, e, f) containing the first two rows from an affine transform matrix. """ + method = Image.AFFINE @@ -67,6 +68,7 @@ class ExtentTransform(Transform): :param bbox: A 4-tuple (x0, y0, x1, y1) which specifies two points in the input image's coordinate system. See :ref:`coordinate-system`. """ + method = Image.EXTENT @@ -83,6 +85,7 @@ class QuadTransform(Transform): upper left, lower left, lower right, and upper right corner of the source quadrilateral. """ + method = Image.QUAD @@ -95,4 +98,5 @@ class MeshTransform(Transform): :param data: A list of (bbox, quad) tuples. """ + method = Image.MESH diff --git a/server/www/packages/packages-linux/x64/PIL/ImageWin.py b/server/www/packages/packages-linux/x64/PIL/ImageWin.py index 9b86270..ed2c18e 100644 --- a/server/www/packages/packages-linux/x64/PIL/ImageWin.py +++ b/server/www/packages/packages-linux/x64/PIL/ImageWin.py @@ -26,6 +26,7 @@ class HDC(object): :py:meth:`~PIL.ImageWin.Dib.draw` and :py:meth:`~PIL.ImageWin.Dib.expose` methods. """ + def __init__(self, dc): self.dc = dc @@ -39,6 +40,7 @@ class HWND(object): :py:meth:`~PIL.ImageWin.Dib.draw` and :py:meth:`~PIL.ImageWin.Dib.expose` methods, instead of a DC. """ + def __init__(self, wnd): self.wnd = wnd @@ -190,7 +192,7 @@ class Window(object): def __init__(self, title="PIL", width=None, height=None): self.hwnd = Image.core.createwindow( title, self.__dispatcher, width or 0, height or 0 - ) + ) def __dispatcher(self, action, *args): return getattr(self, "ui_handle_" + action)(*args) diff --git a/server/www/packages/packages-linux/x64/PIL/ImtImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/ImtImagePlugin.py index 5a6623c..a9e991f 100644 --- a/server/www/packages/packages-linux/x64/PIL/ImtImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/ImtImagePlugin.py @@ -19,6 +19,8 @@ import re from . import Image, ImageFile +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.2" @@ -31,6 +33,7 @@ field = re.compile(br"([a-z]*) ([^ \r\n]*)") ## # Image plugin for IM Tools images. + class ImtImageFile(ImageFile.ImageFile): format = "IMT" @@ -53,12 +56,12 @@ class ImtImageFile(ImageFile.ImageFile): if not s: break - if s == b'\x0C': + if s == b"\x0C": # image data begins - self.tile = [("raw", (0, 0)+self.size, - self.fp.tell(), - (self.mode, 0, 1))] + self.tile = [ + ("raw", (0, 0) + self.size, self.fp.tell(), (self.mode, 0, 1)) + ] break diff --git a/server/www/packages/packages-linux/x64/PIL/IptcImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/IptcImagePlugin.py index b63e1ab..aedf2e4 100644 --- a/server/www/packages/packages-linux/x64/PIL/IptcImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/IptcImagePlugin.py @@ -17,17 +17,17 @@ from __future__ import print_function -from . import Image, ImageFile -from ._binary import i8, i16be as i16, i32be as i32, o8 import os import tempfile +from . import Image, ImageFile +from ._binary import i8, i16be as i16, i32be as i32, o8 + +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.3" -COMPRESSION = { - 1: "raw", - 5: "jpeg" -} +COMPRESSION = {1: "raw", 5: "jpeg"} PAD = o8(0) * 4 @@ -35,13 +35,14 @@ PAD = o8(0) * 4 # # Helpers + def i(c): return i32((PAD + c)[-4:]) def dump(c): for i in c: - print("%02x" % i8(i), end=' ') + print("%02x" % i8(i), end=" ") print() @@ -49,6 +50,7 @@ def dump(c): # Image plugin for IPTC/NAA datastreams. To read IPTC/NAA fields # from TIFF and JPEG files, use the getiptcinfo function. + class IptcImageFile(ImageFile.ImageFile): format = "IPTC" @@ -77,7 +79,7 @@ class IptcImageFile(ImageFile.ImageFile): elif size == 128: size = 0 elif size > 128: - size = i(self.fp.read(size-128)) + size = i(self.fp.read(size - 128)) else: size = i16(s[3:]) @@ -107,7 +109,7 @@ class IptcImageFile(ImageFile.ImageFile): layers = i8(self.info[(3, 60)][0]) component = i8(self.info[(3, 60)][1]) if (3, 65) in self.info: - id = i8(self.info[(3, 65)][0])-1 + id = i8(self.info[(3, 65)][0]) - 1 else: id = 0 if layers == 1 and not component: @@ -128,8 +130,9 @@ class IptcImageFile(ImageFile.ImageFile): # tile if tag == (8, 10): - self.tile = [("iptc", (compression, offset), - (0, 0, self.size[0], self.size[1]))] + self.tile = [ + ("iptc", (compression, offset), (0, 0, self.size[0], self.size[1])) + ] def load(self): @@ -196,35 +199,9 @@ def getiptcinfo(im): elif isinstance(im, JpegImagePlugin.JpegImageFile): # extract the IPTC/NAA resource - try: - app = im.app["APP13"] - if app[:14] == b"Photoshop 3.0\x00": - app = app[14:] - # parse the image resource block - offset = 0 - while app[offset:offset+4] == b"8BIM": - offset += 4 - # resource code - code = i16(app, offset) - offset += 2 - # resource name (usually empty) - name_len = i8(app[offset]) - # name = app[offset+1:offset+1+name_len] - offset = 1 + offset + name_len - if offset & 1: - offset += 1 - # resource data block - size = i32(app, offset) - offset += 4 - if code == 0x0404: - # 0x0404 contains IPTC/NAA data - data = app[offset:offset+size] - break - offset = offset + size - if offset & 1: - offset += 1 - except (AttributeError, KeyError): - pass + photoshop = im.info.get("photoshop") + if photoshop: + data = photoshop.get(0x0404) elif isinstance(im, TiffImagePlugin.TiffImageFile): # get raw data from the IPTC/NAA tag (PhotoShop tags the data @@ -240,6 +217,7 @@ def getiptcinfo(im): # create an IptcImagePlugin object without initializing it class FakeImage(object): pass + im = FakeImage() im.__class__ = IptcImageFile diff --git a/server/www/packages/packages-linux/x64/PIL/Jpeg2KImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/Jpeg2KImagePlugin.py index 7659b6b..37f1117 100644 --- a/server/www/packages/packages-linux/x64/PIL/Jpeg2KImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/Jpeg2KImagePlugin.py @@ -12,11 +12,14 @@ # # See the README file for information on usage and redistribution. # -from . import Image, ImageFile -import struct -import os import io +import os +import struct +from . import Image, ImageFile + +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.1" @@ -25,30 +28,29 @@ def _parse_codestream(fp): count from the SIZ marker segment, returning a PIL (size, mode) tuple.""" hdr = fp.read(2) - lsiz = struct.unpack('>H', hdr)[0] + lsiz = struct.unpack(">H", hdr)[0] siz = hdr + fp.read(lsiz - 2) - lsiz, rsiz, xsiz, ysiz, xosiz, yosiz, xtsiz, ytsiz, \ - xtosiz, ytosiz, csiz \ - = struct.unpack_from('>HHIIIIIIIIH', siz) - ssiz = [None]*csiz - xrsiz = [None]*csiz - yrsiz = [None]*csiz + lsiz, rsiz, xsiz, ysiz, xosiz, yosiz, _, _, _, _, csiz = struct.unpack_from( + ">HHIIIIIIIIH", siz + ) + ssiz = [None] * csiz + xrsiz = [None] * csiz + yrsiz = [None] * csiz for i in range(csiz): - ssiz[i], xrsiz[i], yrsiz[i] \ - = struct.unpack_from('>BBB', siz, 36 + 3 * i) + ssiz[i], xrsiz[i], yrsiz[i] = struct.unpack_from(">BBB", siz, 36 + 3 * i) size = (xsiz - xosiz, ysiz - yosiz) if csiz == 1: - if (yrsiz[0] & 0x7f) > 8: - mode = 'I;16' + if (yrsiz[0] & 0x7F) > 8: + mode = "I;16" else: - mode = 'L' + mode = "L" elif csiz == 2: - mode = 'LA' + mode = "LA" elif csiz == 3: - mode = 'RGB' + mode = "RGB" elif csiz == 4: - mode = 'RGBA' + mode = "RGBA" else: mode = None @@ -57,29 +59,34 @@ def _parse_codestream(fp): def _parse_jp2_header(fp): """Parse the JP2 header box to extract size, component count and - color space information, returning a PIL (size, mode) tuple.""" + color space information, returning a (size, mode, mimetype) tuple.""" # Find the JP2 header box header = None + mimetype = None while True: - lbox, tbox = struct.unpack('>I4s', fp.read(8)) + lbox, tbox = struct.unpack(">I4s", fp.read(8)) if lbox == 1: - lbox = struct.unpack('>Q', fp.read(8))[0] + lbox = struct.unpack(">Q", fp.read(8))[0] hlen = 16 else: hlen = 8 if lbox < hlen: - raise SyntaxError('Invalid JP2 header length') + raise SyntaxError("Invalid JP2 header length") - if tbox == b'jp2h': + if tbox == b"jp2h": header = fp.read(lbox - hlen) break + elif tbox == b"ftyp": + if fp.read(4) == b"jpx ": + mimetype = "image/jpx" + fp.seek(lbox - hlen - 4, os.SEEK_CUR) else: fp.seek(lbox - hlen, os.SEEK_CUR) if header is None: - raise SyntaxError('could not find JP2 header') + raise SyntaxError("could not find JP2 header") size = None mode = None @@ -88,64 +95,64 @@ def _parse_jp2_header(fp): hio = io.BytesIO(header) while True: - lbox, tbox = struct.unpack('>I4s', hio.read(8)) + lbox, tbox = struct.unpack(">I4s", hio.read(8)) if lbox == 1: - lbox = struct.unpack('>Q', hio.read(8))[0] + lbox = struct.unpack(">Q", hio.read(8))[0] hlen = 16 else: hlen = 8 content = hio.read(lbox - hlen) - if tbox == b'ihdr': - height, width, nc, bpc, c, unkc, ipr \ - = struct.unpack('>IIHBBBB', content) + if tbox == b"ihdr": + height, width, nc, bpc, c, unkc, ipr = struct.unpack(">IIHBBBB", content) size = (width, height) if unkc: - if nc == 1 and (bpc & 0x7f) > 8: - mode = 'I;16' + if nc == 1 and (bpc & 0x7F) > 8: + mode = "I;16" elif nc == 1: - mode = 'L' + mode = "L" elif nc == 2: - mode = 'LA' + mode = "LA" elif nc == 3: - mode = 'RGB' + mode = "RGB" elif nc == 4: - mode = 'RGBA' + mode = "RGBA" break - elif tbox == b'colr': - meth, prec, approx = struct.unpack_from('>BBB', content) + elif tbox == b"colr": + meth, prec, approx = struct.unpack_from(">BBB", content) if meth == 1: - cs = struct.unpack_from('>I', content, 3)[0] - if cs == 16: # sRGB - if nc == 1 and (bpc & 0x7f) > 8: - mode = 'I;16' + cs = struct.unpack_from(">I", content, 3)[0] + if cs == 16: # sRGB + if nc == 1 and (bpc & 0x7F) > 8: + mode = "I;16" elif nc == 1: - mode = 'L' + mode = "L" elif nc == 3: - mode = 'RGB' + mode = "RGB" elif nc == 4: - mode = 'RGBA' + mode = "RGBA" break elif cs == 17: # grayscale - if nc == 1 and (bpc & 0x7f) > 8: - mode = 'I;16' + if nc == 1 and (bpc & 0x7F) > 8: + mode = "I;16" elif nc == 1: - mode = 'L' + mode = "L" elif nc == 2: - mode = 'LA' + mode = "LA" break elif cs == 18: # sYCC if nc == 3: - mode = 'RGB' + mode = "RGB" elif nc == 4: - mode = 'RGBA' + mode = "RGBA" break if size is None or mode is None: raise SyntaxError("Malformed jp2 header") - return (size, mode) + return (size, mode, mimetype) + ## # Image plugin for JPEG2000 images. @@ -157,20 +164,21 @@ class Jpeg2KImageFile(ImageFile.ImageFile): def _open(self): sig = self.fp.read(4) - if sig == b'\xff\x4f\xff\x51': + if sig == b"\xff\x4f\xff\x51": self.codec = "j2k" self._size, self.mode = _parse_codestream(self.fp) else: sig = sig + self.fp.read(8) - if sig == b'\x00\x00\x00\x0cjP \x0d\x0a\x87\x0a': + if sig == b"\x00\x00\x00\x0cjP \x0d\x0a\x87\x0a": self.codec = "jp2" - self._size, self.mode = _parse_jp2_header(self.fp) + header = _parse_jp2_header(self.fp) + self._size, self.mode, self.custom_mimetype = header else: - raise SyntaxError('not a JPEG 2000 file') + raise SyntaxError("not a JPEG 2000 file") if self.size is None or self.mode is None: - raise SyntaxError('unable to determine size/mode') + raise SyntaxError("unable to determine size/mode") self.reduce = 0 self.layers = 0 @@ -181,25 +189,33 @@ class Jpeg2KImageFile(ImageFile.ImageFile): try: fd = self.fp.fileno() length = os.fstat(fd).st_size - except: + except Exception: fd = -1 try: pos = self.fp.tell() - self.fp.seek(0, 2) + self.fp.seek(0, io.SEEK_END) length = self.fp.tell() - self.fp.seek(pos, 0) - except: + self.fp.seek(pos) + except Exception: length = -1 - self.tile = [('jpeg2k', (0, 0) + self.size, 0, - (self.codec, self.reduce, self.layers, fd, length))] + self.tile = [ + ( + "jpeg2k", + (0, 0) + self.size, + 0, + (self.codec, self.reduce, self.layers, fd, length), + ) + ] def load(self): if self.reduce: power = 1 << self.reduce adjust = power >> 1 - self._size = (int((self.size[0] + adjust) / power), - int((self.size[1] + adjust) / power)) + self._size = ( + int((self.size[0] + adjust) / power), + int((self.size[1] + adjust) / power), + ) if self.tile: # Update the reduce and layers settings @@ -211,39 +227,53 @@ class Jpeg2KImageFile(ImageFile.ImageFile): def _accept(prefix): - return (prefix[:4] == b'\xff\x4f\xff\x51' or - prefix[:12] == b'\x00\x00\x00\x0cjP \x0d\x0a\x87\x0a') + return ( + prefix[:4] == b"\xff\x4f\xff\x51" + or prefix[:12] == b"\x00\x00\x00\x0cjP \x0d\x0a\x87\x0a" + ) # ------------------------------------------------------------ # Save support + def _save(im, fp, filename): - if filename.endswith('.j2k'): - kind = 'j2k' + if filename.endswith(".j2k"): + kind = "j2k" else: - kind = 'jp2' + kind = "jp2" # Get the keyword arguments info = im.encoderinfo - offset = info.get('offset', None) - tile_offset = info.get('tile_offset', None) - tile_size = info.get('tile_size', None) - quality_mode = info.get('quality_mode', 'rates') - quality_layers = info.get('quality_layers', None) - num_resolutions = info.get('num_resolutions', 0) - cblk_size = info.get('codeblock_size', None) - precinct_size = info.get('precinct_size', None) - irreversible = info.get('irreversible', False) - progression = info.get('progression', 'LRCP') - cinema_mode = info.get('cinema_mode', 'no') + offset = info.get("offset", None) + tile_offset = info.get("tile_offset", None) + tile_size = info.get("tile_size", None) + quality_mode = info.get("quality_mode", "rates") + quality_layers = info.get("quality_layers", None) + if quality_layers is not None and not ( + isinstance(quality_layers, (list, tuple)) + and all( + [ + isinstance(quality_layer, (int, float)) + for quality_layer in quality_layers + ] + ) + ): + raise ValueError("quality_layers must be a sequence of numbers") + + num_resolutions = info.get("num_resolutions", 0) + cblk_size = info.get("codeblock_size", None) + precinct_size = info.get("precinct_size", None) + irreversible = info.get("irreversible", False) + progression = info.get("progression", "LRCP") + cinema_mode = info.get("cinema_mode", "no") fd = -1 if hasattr(fp, "fileno"): try: fd = fp.fileno() - except: + except Exception: fd = -1 im.encoderconfig = ( @@ -258,10 +288,11 @@ def _save(im, fp, filename): irreversible, progression, cinema_mode, - fd + fd, ) - ImageFile._save(im, fp, [('jpeg2k', (0, 0)+im.size, 0, kind)]) + ImageFile._save(im, fp, [("jpeg2k", (0, 0) + im.size, 0, kind)]) + # ------------------------------------------------------------ # Registry stuff @@ -270,8 +301,8 @@ def _save(im, fp, filename): Image.register_open(Jpeg2KImageFile.format, Jpeg2KImageFile, _accept) Image.register_save(Jpeg2KImageFile.format, _save) -Image.register_extensions(Jpeg2KImageFile.format, - [".jp2", ".j2k", ".jpc", ".jpf", ".jpx", ".j2c"]) +Image.register_extensions( + Jpeg2KImageFile.format, [".jp2", ".j2k", ".jpc", ".jpf", ".jpx", ".j2c"] +) -Image.register_mime(Jpeg2KImageFile.format, 'image/jp2') -Image.register_mime(Jpeg2KImageFile.format, 'image/jpx') +Image.register_mime(Jpeg2KImageFile.format, "image/jp2") diff --git a/server/www/packages/packages-linux/x64/PIL/JpegImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/JpegImagePlugin.py index f206818..020b952 100644 --- a/server/www/packages/packages-linux/x64/PIL/JpegImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/JpegImagePlugin.py @@ -35,22 +35,26 @@ from __future__ import print_function import array -import struct import io +import struct import warnings -from . import Image, ImageFile, TiffImagePlugin -from ._binary import i8, o8, i16be as i16 -from .JpegPresets import presets -from ._util import isStringType +from . import Image, ImageFile, TiffImagePlugin +from ._binary import i8, i16be as i16, i32be as i32, o8 +from ._util import isStringType +from .JpegPresets import presets + +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.6" # # Parser + def Skip(self, marker): - n = i16(self.fp.read(2))-2 + n = i16(self.fp.read(2)) - 2 ImageFile._safe_read(self.fp, n) @@ -59,7 +63,7 @@ def APP(self, marker): # Application marker. Store these in the APP dictionary. # Also look for well-known application markers. - n = i16(self.fp.read(2))-2 + n = i16(self.fp.read(2)) - 2 s = ImageFile._safe_read(self.fp, n) app = "APP%d" % (marker & 15) @@ -75,7 +79,7 @@ def APP(self, marker): try: jfif_unit = i8(s[7]) jfif_density = i16(s, 8), i16(s, 10) - except: + except Exception: pass else: if jfif_unit == 1: @@ -84,7 +88,7 @@ def APP(self, marker): self.info["jfif_density"] = jfif_density elif marker == 0xFFE1 and s[:5] == b"Exif\0": if "exif" not in self.info: - # extract Exif information (incomplete) + # extract EXIF information (incomplete) self.info["exif"] = s # FIXME: value will change elif marker == 0xFFE2 and s[:5] == b"FPXR\0": # extract FlashPix information (incomplete) @@ -102,12 +106,45 @@ def APP(self, marker): # reassemble the profile, rather than assuming that the APP2 # markers appear in the correct sequence. self.icclist.append(s) + elif marker == 0xFFED: + if s[:14] == b"Photoshop 3.0\x00": + blocks = s[14:] + # parse the image resource block + offset = 0 + photoshop = {} + while blocks[offset : offset + 4] == b"8BIM": + offset += 4 + # resource code + code = i16(blocks, offset) + offset += 2 + # resource name (usually empty) + name_len = i8(blocks[offset]) + # name = blocks[offset+1:offset+1+name_len] + offset = 1 + offset + name_len + if offset & 1: + offset += 1 + # resource data block + size = i32(blocks, offset) + offset += 4 + data = blocks[offset : offset + size] + if code == 0x03ED: # ResolutionInfo + data = { + "XResolution": i32(data[:4]) / 65536, + "DisplayedUnitsX": i16(data[4:8]), + "YResolution": i32(data[8:12]) / 65536, + "DisplayedUnitsY": i16(data[12:]), + } + photoshop[code] = data + offset = offset + size + if offset & 1: + offset += 1 + self.info["photoshop"] = photoshop elif marker == 0xFFEE and s[:5] == b"Adobe": self.info["adobe"] = i16(s, 5) # extract Adobe custom properties try: adobe_transform = i8(s[1]) - except: + except Exception: pass else: self.info["adobe_transform"] = adobe_transform @@ -121,19 +158,19 @@ def APP(self, marker): # If DPI isn't in JPEG header, fetch from EXIF if "dpi" not in self.info and "exif" in self.info: try: - exif = self._getexif() + exif = self.getexif() resolution_unit = exif[0x0128] x_resolution = exif[0x011A] try: - dpi = x_resolution[0] / x_resolution[1] + dpi = float(x_resolution[0]) / x_resolution[1] except TypeError: dpi = x_resolution if resolution_unit == 3: # cm # 1 dpcm = 2.54 dpi dpi *= 2.54 - self.info["dpi"] = dpi, dpi + self.info["dpi"] = int(dpi + 0.5), int(dpi + 0.5) except (KeyError, SyntaxError, ZeroDivisionError): - # SyntaxError for invalid/unreadable exif + # SyntaxError for invalid/unreadable EXIF # KeyError for dpi not included # ZeroDivisionError for invalid dpi rational value self.info["dpi"] = 72, 72 @@ -142,7 +179,7 @@ def APP(self, marker): def COM(self, marker): # # Comment marker. Store these in the APP dictionary. - n = i16(self.fp.read(2))-2 + n = i16(self.fp.read(2)) - 2 s = ImageFile._safe_read(self.fp, n) self.app["COM"] = s # compatibility @@ -157,7 +194,7 @@ def SOF(self, marker): # mode. Note that this could be made a bit brighter, by # looking for JFIF and Adobe APP markers. - n = i16(self.fp.read(2))-2 + n = i16(self.fp.read(2)) - 2 s = ImageFile._safe_read(self.fp, n) self._size = i16(s[3:]), i16(s[1:]) @@ -192,9 +229,9 @@ def SOF(self, marker): self.icclist = None for i in range(6, len(s), 3): - t = s[i:i+3] + t = s[i : i + 3] # 4-tuples: id, vsamp, hsamp, qtable - self.layer.append((t[0], i8(t[1])//16, i8(t[1]) & 15, i8(t[2]))) + self.layer.append((t[0], i8(t[1]) // 16, i8(t[1]) & 15, i8(t[2]))) def DQT(self, marker): @@ -206,13 +243,13 @@ def DQT(self, marker): # FIXME: The quantization tables can be used to estimate the # compression quality. - n = i16(self.fp.read(2))-2 + n = i16(self.fp.read(2)) - 2 s = ImageFile._safe_read(self.fp, n) while len(s): if len(s) < 65: raise SyntaxError("bad quantization table marker") v = i8(s[0]) - if v//16 == 0: + if v // 16 == 0: self.quantization[v & 15] = array.array("B", s[1:65]) s = s[65:] else: @@ -286,7 +323,7 @@ MARKER = { 0xFFFB: ("JPG11", "Extension 11", None), 0xFFFC: ("JPG12", "Extension 12", None), 0xFFFD: ("JPG13", "Extension 13", None), - 0xFFFE: ("COM", "Comment", COM) + 0xFFFE: ("COM", "Comment", COM), } @@ -297,6 +334,7 @@ def _accept(prefix): ## # Image plugin for JPEG and JFIF images. + class JpegImageFile(ImageFile.ImageFile): format = "JPEG" @@ -340,8 +378,7 @@ class JpegImageFile(ImageFile.ImageFile): rawmode = self.mode if self.mode == "CMYK": rawmode = "CMYK;I" # assume adobe conventions - self.tile = [("jpeg", (0, 0) + self.size, 0, - (rawmode, ""))] + self.tile = [("jpeg", (0, 0) + self.size, 0, (rawmode, ""))] # self.__offset = self.fp.tell() break s = self.fp.read(1) @@ -389,8 +426,13 @@ class JpegImageFile(ImageFile.ImageFile): for s in [8, 4, 2, 1]: if scale >= s: break - e = e[0], e[1], (e[2]-e[0]+s-1)//s+e[0], (e[3]-e[1]+s-1)//s+e[1] - self._size = ((self.size[0]+s-1)//s, (self.size[1]+s-1)//s) + e = ( + e[0], + e[1], + (e[2] - e[0] + s - 1) // s + e[0], + (e[3] - e[1] + s - 1) // s + e[1], + ) + self._size = ((self.size[0] + s - 1) // s, (self.size[1] + s - 1) // s) scale = s self.tile = [(d, e, o, a)] @@ -405,6 +447,7 @@ class JpegImageFile(ImageFile.ImageFile): import subprocess import tempfile import os + f, path = tempfile.mkstemp() os.close(f) if os.path.exists(self.filename): @@ -437,60 +480,14 @@ class JpegImageFile(ImageFile.ImageFile): def _fixup_dict(src_dict): # Helper function for _getexif() # returns a dict with any single item tuples/lists as individual values - def _fixup(value): - try: - if len(value) == 1 and not isinstance(value, dict): - return value[0] - except: - pass - return value - - return {k: _fixup(v) for k, v in src_dict.items()} + exif = Image.Exif() + return exif._fixup_dict(src_dict) def _getexif(self): - # Extract EXIF information. This method is highly experimental, - # and is likely to be replaced with something better in a future - # version. - - # The EXIF record consists of a TIFF file embedded in a JPEG - # application marker (!). - try: - data = self.info["exif"] - except KeyError: + if "exif" not in self.info: return None - file = io.BytesIO(data[6:]) - head = file.read(8) - # process dictionary - info = TiffImagePlugin.ImageFileDirectory_v1(head) - info.load(file) - exif = dict(_fixup_dict(info)) - # get exif extension - try: - # exif field 0x8769 is an offset pointer to the location - # of the nested embedded exif ifd. - # It should be a long, but may be corrupted. - file.seek(exif[0x8769]) - except (KeyError, TypeError): - pass - else: - info = TiffImagePlugin.ImageFileDirectory_v1(head) - info.load(file) - exif.update(_fixup_dict(info)) - # get gpsinfo extension - try: - # exif field 0x8825 is an offset pointer to the location - # of the nested embedded gps exif ifd. - # It should be a long, but may be corrupted. - file.seek(exif[0x8825]) - except (KeyError, TypeError): - pass - else: - info = TiffImagePlugin.ImageFileDirectory_v1(head) - info.load(file) - exif[0x8825] = _fixup_dict(info) - - return exif + return dict(self.getexif()) def _getmp(self): @@ -506,13 +503,14 @@ def _getmp(self): return None file_contents = io.BytesIO(data) head = file_contents.read(8) - endianness = '>' if head[:4] == b'\x4d\x4d\x00\x2a' else '<' + endianness = ">" if head[:4] == b"\x4d\x4d\x00\x2a" else "<" # process dictionary try: info = TiffImagePlugin.ImageFileDirectory_v2(head) + file_contents.seek(info.next) info.load(file_contents) mp = dict(info) - except: + except Exception: raise SyntaxError("malformed MP Index (unreadable directory)") # it's an error not to have a number of images try: @@ -525,37 +523,33 @@ def _getmp(self): rawmpentries = mp[0xB002] for entrynum in range(0, quant): unpackedentry = struct.unpack_from( - '{}LLLHH'.format(endianness), rawmpentries, entrynum * 16) - labels = ('Attribute', 'Size', 'DataOffset', 'EntryNo1', - 'EntryNo2') + "{}LLLHH".format(endianness), rawmpentries, entrynum * 16 + ) + labels = ("Attribute", "Size", "DataOffset", "EntryNo1", "EntryNo2") mpentry = dict(zip(labels, unpackedentry)) mpentryattr = { - 'DependentParentImageFlag': bool(mpentry['Attribute'] & - (1 << 31)), - 'DependentChildImageFlag': bool(mpentry['Attribute'] & - (1 << 30)), - 'RepresentativeImageFlag': bool(mpentry['Attribute'] & - (1 << 29)), - 'Reserved': (mpentry['Attribute'] & (3 << 27)) >> 27, - 'ImageDataFormat': (mpentry['Attribute'] & (7 << 24)) >> 24, - 'MPType': mpentry['Attribute'] & 0x00FFFFFF + "DependentParentImageFlag": bool(mpentry["Attribute"] & (1 << 31)), + "DependentChildImageFlag": bool(mpentry["Attribute"] & (1 << 30)), + "RepresentativeImageFlag": bool(mpentry["Attribute"] & (1 << 29)), + "Reserved": (mpentry["Attribute"] & (3 << 27)) >> 27, + "ImageDataFormat": (mpentry["Attribute"] & (7 << 24)) >> 24, + "MPType": mpentry["Attribute"] & 0x00FFFFFF, } - if mpentryattr['ImageDataFormat'] == 0: - mpentryattr['ImageDataFormat'] = 'JPEG' + if mpentryattr["ImageDataFormat"] == 0: + mpentryattr["ImageDataFormat"] = "JPEG" else: raise SyntaxError("unsupported picture format in MPO") mptypemap = { - 0x000000: 'Undefined', - 0x010001: 'Large Thumbnail (VGA Equivalent)', - 0x010002: 'Large Thumbnail (Full HD Equivalent)', - 0x020001: 'Multi-Frame Image (Panorama)', - 0x020002: 'Multi-Frame Image: (Disparity)', - 0x020003: 'Multi-Frame Image: (Multi-Angle)', - 0x030000: 'Baseline MP Primary Image' + 0x000000: "Undefined", + 0x010001: "Large Thumbnail (VGA Equivalent)", + 0x010002: "Large Thumbnail (Full HD Equivalent)", + 0x020001: "Multi-Frame Image (Panorama)", + 0x020002: "Multi-Frame Image: (Disparity)", + 0x020003: "Multi-Frame Image: (Multi-Angle)", + 0x030000: "Baseline MP Primary Image", } - mpentryattr['MPType'] = mptypemap.get(mpentryattr['MPType'], - 'Unknown') - mpentry['Attribute'] = mpentryattr + mpentryattr["MPType"] = mptypemap.get(mpentryattr["MPType"], "Unknown") + mpentry["Attribute"] = mpentryattr mpentries.append(mpentry) mp[0xB002] = mpentries except KeyError: @@ -578,19 +572,24 @@ RAWMODE = { "YCbCr": "YCbCr", } -zigzag_index = (0, 1, 5, 6, 14, 15, 27, 28, - 2, 4, 7, 13, 16, 26, 29, 42, - 3, 8, 12, 17, 25, 30, 41, 43, - 9, 11, 18, 24, 31, 40, 44, 53, - 10, 19, 23, 32, 39, 45, 52, 54, - 20, 22, 33, 38, 46, 51, 55, 60, - 21, 34, 37, 47, 50, 56, 59, 61, - 35, 36, 48, 49, 57, 58, 62, 63) +# fmt: off +zigzag_index = ( + 0, 1, 5, 6, 14, 15, 27, 28, + 2, 4, 7, 13, 16, 26, 29, 42, + 3, 8, 12, 17, 25, 30, 41, 43, + 9, 11, 18, 24, 31, 40, 44, 53, + 10, 19, 23, 32, 39, 45, 52, 54, + 20, 22, 33, 38, 46, 51, 55, 60, + 21, 34, 37, 47, 50, 56, 59, 61, + 35, 36, 48, 49, 57, 58, 62, 63, +) -samplings = {(1, 1, 1, 1, 1, 1): 0, - (2, 1, 1, 1, 1, 1): 1, - (2, 2, 1, 1, 1, 1): 2, - } +samplings = { + (1, 1, 1, 1, 1, 1): 0, + (2, 1, 1, 1, 1, 1): 1, + (2, 2, 1, 1, 1, 1): 2, +} +# fmt: on def convert_dict_qtables(qtables): @@ -608,7 +607,7 @@ def get_sampling(im): # NOTE: currently Pillow can't encode JPEG to YCCK format. # If YCCK support is added in the future, subsampling code will have # to be updated (here and in JpegEncode.c) to deal with 4 layers. - if not hasattr(im, 'layers') or im.layers in (1, 4): + if not hasattr(im, "layers") or im.layers in (1, 4): return -1 sampling = im.layer[0][1:3] + im.layer[1][1:3] + im.layer[2][1:3] return samplings.get(sampling, -1) @@ -636,15 +635,15 @@ def _save(im, fp, filename): elif quality in presets: preset = presets[quality] quality = 0 - subsampling = preset.get('subsampling', -1) - qtables = preset.get('quantization') + subsampling = preset.get("subsampling", -1) + qtables = preset.get("quantization") elif not isinstance(quality, int): raise ValueError("Invalid quality setting") else: if subsampling in presets: - subsampling = presets[subsampling].get('subsampling', -1) + subsampling = presets[subsampling].get("subsampling", -1) if isStringType(qtables) and qtables in presets: - qtables = presets[qtables].get('quantization') + qtables = presets[qtables].get("quantization") if subsampling == "4:4:4": subsampling = 0 @@ -658,8 +657,7 @@ def _save(im, fp, filename): subsampling = 2 elif subsampling == "keep": if im.format != "JPEG": - raise ValueError( - "Cannot use 'keep' when original image is not a JPEG") + raise ValueError("Cannot use 'keep' when original image is not a JPEG") subsampling = get_sampling(im) def validate_qtables(qtables): @@ -667,12 +665,15 @@ def _save(im, fp, filename): return qtables if isStringType(qtables): try: - lines = [int(num) for line in qtables.splitlines() - for num in line.split('#', 1)[0].split()] + lines = [ + int(num) + for line in qtables.splitlines() + for num in line.split("#", 1)[0].split() + ] except ValueError: raise ValueError("Invalid quantization table") else: - qtables = [lines[s:s+64] for s in range(0, len(lines), 64)] + qtables = [lines[s : s + 64] for s in range(0, len(lines), 64)] if isinstance(qtables, (tuple, list, dict)): if isinstance(qtables, dict): qtables = convert_dict_qtables(qtables) @@ -684,7 +685,7 @@ def _save(im, fp, filename): try: if len(table) != 64: raise TypeError - table = array.array('B', table) + table = array.array("B", table) except TypeError: raise ValueError("Invalid quantization table") else: @@ -693,8 +694,7 @@ def _save(im, fp, filename): if qtables == "keep": if im.format != "JPEG": - raise ValueError( - "Cannot use 'keep' when original image is not a JPEG") + raise ValueError("Cannot use 'keep' when original image is not a JPEG") qtables = getattr(im, "quantization", None) qtables = validate_qtables(qtables) @@ -712,18 +712,27 @@ def _save(im, fp, filename): i = 1 for marker in markers: size = struct.pack(">H", 2 + ICC_OVERHEAD_LEN + len(marker)) - extra += (b"\xFF\xE2" + size + b"ICC_PROFILE\0" + o8(i) + - o8(len(markers)) + marker) + extra += ( + b"\xFF\xE2" + + size + + b"ICC_PROFILE\0" + + o8(i) + + o8(len(markers)) + + marker + ) i += 1 # "progressive" is the official name, but older documentation # says "progression" # FIXME: issue a warning if the wrong form is used (post-1.1.7) - progressive = (info.get("progressive", False) or - info.get("progression", False)) + progressive = info.get("progressive", False) or info.get("progression", False) optimize = info.get("optimize", False) + exif = info.get("exif", b"") + if isinstance(exif, Image.Exif): + exif = exif.tobytes() + # get keyword arguments im.encoderconfig = ( quality, @@ -731,12 +740,13 @@ def _save(im, fp, filename): info.get("smooth", 0), optimize, info.get("streamtype", 0), - dpi[0], dpi[1], + dpi[0], + dpi[1], subsampling, qtables, extra, - info.get("exif", b"") - ) + exif, + ) # if we optimize, libjpeg needs a buffer big enough to hold the whole image # in a shot. Guessing on the size, at im.size bytes. (raw pixel size is @@ -745,7 +755,7 @@ def _save(im, fp, filename): bufsize = 0 if optimize or progressive: # CMYK can be bigger - if im.mode == 'CMYK': + if im.mode == "CMYK": bufsize = 4 * im.size[0] * im.size[1] # keep sets quality to 0, but the actual value may be high. elif quality >= 95 or quality == 0: @@ -753,18 +763,18 @@ def _save(im, fp, filename): else: bufsize = im.size[0] * im.size[1] - # The exif info needs to be written as one block, + APP1, + one spare byte. + # The EXIF info needs to be written as one block, + APP1, + one spare byte. # Ensure that our buffer is big enough. Same with the icc_profile block. - bufsize = max(ImageFile.MAXBLOCK, bufsize, len(info.get("exif", b"")) + 5, - len(extra) + 1) + bufsize = max(ImageFile.MAXBLOCK, bufsize, len(exif) + 5, len(extra) + 1) - ImageFile._save(im, fp, [("jpeg", (0, 0)+im.size, 0, rawmode)], bufsize) + ImageFile._save(im, fp, [("jpeg", (0, 0) + im.size, 0, rawmode)], bufsize) def _save_cjpeg(im, fp, filename): # ALTERNATIVE: handle JPEGs via the IJG command line utilities. import os import subprocess + tempfile = im._dump() subprocess.check_call(["cjpeg", "-outfile", filename, tempfile]) try: @@ -782,13 +792,17 @@ def jpeg_factory(fp=None, filename=None): if mpheader[45057] > 1: # It's actually an MPO from .MpoImagePlugin import MpoImageFile - im = MpoImageFile(fp, filename) + + # Don't reload everything, just convert it. + im = MpoImageFile.adopt(im, mpheader) except (TypeError, IndexError): # It is really a JPEG pass except SyntaxError: - warnings.warn("Image appears to be a malformed MPO file, it will be " - "interpreted as a base JPEG file") + warnings.warn( + "Image appears to be a malformed MPO file, it will be " + "interpreted as a base JPEG file" + ) return im @@ -798,7 +812,6 @@ def jpeg_factory(fp=None, filename=None): Image.register_open(JpegImageFile.format, jpeg_factory, _accept) Image.register_save(JpegImageFile.format, _save) -Image.register_extensions(JpegImageFile.format, - [".jfif", ".jpe", ".jpg", ".jpeg"]) +Image.register_extensions(JpegImageFile.format, [".jfif", ".jpe", ".jpg", ".jpeg"]) Image.register_mime(JpegImageFile.format, "image/jpeg") diff --git a/server/www/packages/packages-linux/x64/PIL/JpegPresets.py b/server/www/packages/packages-linux/x64/PIL/JpegPresets.py index 5f01f0d..387844f 100644 --- a/server/www/packages/packages-linux/x64/PIL/JpegPresets.py +++ b/server/www/packages/packages-linux/x64/PIL/JpegPresets.py @@ -62,11 +62,13 @@ The tables format between im.quantization and quantization in presets differ in You can convert the dict format to the preset format with the `JpegImagePlugin.convert_dict_qtables(dict_qtables)` function. -Libjpeg ref.: https://web.archive.org/web/20120328125543/http://www.jpegcameras.com/libjpeg/libjpeg-3.html +Libjpeg ref.: +https://web.archive.org/web/20120328125543/http://www.jpegcameras.com/libjpeg/libjpeg-3.html """ -presets = { +# fmt: off +presets = { # noqa: E128 'web_low': {'subsampling': 2, # "4:2:0" 'quantization': [ [20, 16, 25, 39, 50, 46, 62, 68, @@ -239,3 +241,4 @@ presets = { 15, 12, 12, 12, 12, 12, 12, 12] ]}, } +# fmt: on diff --git a/server/www/packages/packages-linux/x64/PIL/McIdasImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/McIdasImagePlugin.py index 161fb5e..bddd33a 100644 --- a/server/www/packages/packages-linux/x64/PIL/McIdasImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/McIdasImagePlugin.py @@ -17,8 +17,11 @@ # import struct + from . import Image, ImageFile +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.2" @@ -29,6 +32,7 @@ def _accept(s): ## # Image plugin for McIdas area images. + class McIdasImageFile(ImageFile.ImageFile): format = "MCIDAS" @@ -62,7 +66,7 @@ class McIdasImageFile(ImageFile.ImageFile): self._size = w[10], w[9] offset = w[34] + w[15] - stride = w[15] + w[10]*w[11]*w[14] + stride = w[15] + w[10] * w[11] * w[14] self.tile = [("raw", (0, 0) + self.size, offset, (rawmode, stride, 1))] diff --git a/server/www/packages/packages-linux/x64/PIL/MicImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/MicImagePlugin.py index 1dbb6a5..b48905b 100644 --- a/server/www/packages/packages-linux/x64/PIL/MicImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/MicImagePlugin.py @@ -17,10 +17,12 @@ # -from . import Image, TiffImagePlugin - import olefile +from . import Image, TiffImagePlugin + +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.1" @@ -35,6 +37,7 @@ def _accept(prefix): ## # Image plugin for Microsoft's Image Composer file format. + class MicImageFile(TiffImagePlugin.TiffImageFile): format = "MIC" @@ -95,9 +98,17 @@ class MicImageFile(TiffImagePlugin.TiffImageFile): self.frame = frame def tell(self): - return self.frame + def _close__fp(self): + try: + if self.__fp != self.fp: + self.__fp.close() + except AttributeError: + pass + finally: + self.__fp = None + # # -------------------------------------------------------------------- diff --git a/server/www/packages/packages-linux/x64/PIL/MpegImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/MpegImagePlugin.py index 15c7afc..9c662fc 100644 --- a/server/www/packages/packages-linux/x64/PIL/MpegImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/MpegImagePlugin.py @@ -17,14 +17,16 @@ from . import Image, ImageFile from ._binary import i8 +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.1" # # Bitstream parser -class BitStream(object): +class BitStream(object): def __init__(self, fp): self.fp = fp self.bits = 0 @@ -59,6 +61,7 @@ class BitStream(object): # Image plugin for MPEG streams. This plugin can identify a stream, # but it cannot read it. + class MpegImageFile(ImageFile.ImageFile): format = "MPEG" diff --git a/server/www/packages/packages-linux/x64/PIL/MpoImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/MpoImagePlugin.py index a1a8d65..938f2a5 100644 --- a/server/www/packages/packages-linux/x64/PIL/MpoImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/MpoImagePlugin.py @@ -18,8 +18,11 @@ # See the README file for information on usage and redistribution. # -from . import Image, JpegImagePlugin +from . import Image, ImageFile, JpegImagePlugin +from ._binary import i16be as i16 +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.1" @@ -35,6 +38,7 @@ def _save(im, fp, filename): ## # Image plugin for MPO images. + class MpoImageFile(JpegImagePlugin.JpegImageFile): format = "MPO" @@ -44,15 +48,19 @@ class MpoImageFile(JpegImagePlugin.JpegImageFile): def _open(self): self.fp.seek(0) # prep the fp in order to pass the JPEG test JpegImagePlugin.JpegImageFile._open(self) - self.mpinfo = self._getmp() + self._after_jpeg_open() + + def _after_jpeg_open(self, mpheader=None): + self.mpinfo = mpheader if mpheader is not None else self._getmp() self.__framecount = self.mpinfo[0xB001] - self.__mpoffsets = [mpent['DataOffset'] + self.info['mpoffset'] - for mpent in self.mpinfo[0xB002]] + self.__mpoffsets = [ + mpent["DataOffset"] + self.info["mpoffset"] for mpent in self.mpinfo[0xB002] + ] self.__mpoffsets[0] = 0 # Note that the following assertion will only be invalid if something # gets broken within JpegImagePlugin. assert self.__framecount == len(self.__mpoffsets) - del self.info['mpoffset'] # no longer needed + del self.info["mpoffset"] # no longer needed self.__fp = self.fp # FIXME: hack self.__fp.seek(self.__mpoffsets[0]) # get ready to read first frame self.__frame = 0 @@ -76,14 +84,49 @@ class MpoImageFile(JpegImagePlugin.JpegImageFile): return self.fp = self.__fp self.offset = self.__mpoffsets[frame] - self.tile = [ - ("jpeg", (0, 0) + self.size, self.offset, (self.mode, "")) - ] + + self.fp.seek(self.offset + 2) # skip SOI marker + if i16(self.fp.read(2)) == 0xFFE1: # APP1 + n = i16(self.fp.read(2)) - 2 + self.info["exif"] = ImageFile._safe_read(self.fp, n) + + exif = self.getexif() + if 40962 in exif and 40963 in exif: + self._size = (exif[40962], exif[40963]) + elif "exif" in self.info: + del self.info["exif"] + + self.tile = [("jpeg", (0, 0) + self.size, self.offset, (self.mode, ""))] self.__frame = frame def tell(self): return self.__frame + def _close__fp(self): + try: + if self.__fp != self.fp: + self.__fp.close() + except AttributeError: + pass + finally: + self.__fp = None + + @staticmethod + def adopt(jpeg_instance, mpheader=None): + """ + Transform the instance of JpegImageFile into + an instance of MpoImageFile. + After the call, the JpegImageFile is extended + to be an MpoImageFile. + + This is essentially useful when opening a JPEG + file that reveals itself as an MPO, to avoid + double call to _open. + """ + jpeg_instance.__class__ = MpoImageFile + jpeg_instance._after_jpeg_open(mpheader) + return jpeg_instance + # --------------------------------------------------------------------- # Registry stuff diff --git a/server/www/packages/packages-linux/x64/PIL/MspImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/MspImagePlugin.py index 74c6817..7315ab6 100644 --- a/server/www/packages/packages-linux/x64/PIL/MspImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/MspImagePlugin.py @@ -23,11 +23,14 @@ # # See also: http://www.fileformat.info/format/mspaint/egff.htm -from . import Image, ImageFile -from ._binary import i16le as i16, o16le as o16, i8 -import struct import io +import struct +from . import Image, ImageFile +from ._binary import i8, i16le as i16, o16le as o16 + +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.1" @@ -43,6 +46,7 @@ def _accept(prefix): # Image plugin for Windows MSP images. This plugin supports both # uncompressed (Windows 1.0). + class MspImageFile(ImageFile.ImageFile): format = "MSP" @@ -58,7 +62,7 @@ class MspImageFile(ImageFile.ImageFile): # Header checksum checksum = 0 for i in range(0, 32, 2): - checksum = checksum ^ i16(s[i:i+2]) + checksum = checksum ^ i16(s[i : i + 2]) if checksum != 0: raise SyntaxError("bad MSP checksum") @@ -66,9 +70,9 @@ class MspImageFile(ImageFile.ImageFile): self._size = i16(s[4:]), i16(s[6:]) if s[:4] == b"DanM": - self.tile = [("raw", (0, 0)+self.size, 32, ("1", 0, 1))] + self.tile = [("raw", (0, 0) + self.size, 32, ("1", 0, 1))] else: - self.tile = [("MSP", (0, 0)+self.size, 32, None)] + self.tile = [("MSP", (0, 0) + self.size, 32, None)] class MspDecoder(ImageFile.PyDecoder): @@ -111,11 +115,12 @@ class MspDecoder(ImageFile.PyDecoder): def decode(self, buffer): img = io.BytesIO() - blank_line = bytearray((0xff,)*((self.state.xsize+7)//8)) + blank_line = bytearray((0xFF,) * ((self.state.xsize + 7) // 8)) try: self.fd.seek(32) - rowmap = struct.unpack_from("<%dH" % (self.state.ysize), - self.fd.read(self.state.ysize*2)) + rowmap = struct.unpack_from( + "<%dH" % (self.state.ysize), self.fd.read(self.state.ysize * 2) + ) except struct.error: raise IOError("Truncated MSP file in row map") @@ -127,8 +132,8 @@ class MspDecoder(ImageFile.PyDecoder): row = self.fd.read(rowlen) if len(row) != rowlen: raise IOError( - "Truncated MSP file, expected %d bytes on row %s", - (rowlen, x)) + "Truncated MSP file, expected %d bytes on row %s", (rowlen, x) + ) idx = 0 while idx < rowlen: runtype = i8(row[idx]) @@ -139,7 +144,7 @@ class MspDecoder(ImageFile.PyDecoder): idx += 2 else: runcount = runtype - img.write(row[idx:idx+runcount]) + img.write(row[idx : idx + runcount]) idx += runcount except struct.error: @@ -150,7 +155,7 @@ class MspDecoder(ImageFile.PyDecoder): return 0, 0 -Image.register_decoder('MSP', MspDecoder) +Image.register_decoder("MSP", MspDecoder) # @@ -181,7 +186,7 @@ def _save(im, fp, filename): fp.write(o16(h)) # image body - ImageFile._save(im, fp, [("raw", (0, 0)+im.size, 32, ("1", 0, 1))]) + ImageFile._save(im, fp, [("raw", (0, 0) + im.size, 32, ("1", 0, 1))]) # diff --git a/server/www/packages/packages-linux/x64/PIL/OleFileIO.py b/server/www/packages/packages-linux/x64/PIL/OleFileIO.py deleted file mode 100644 index b3caa10..0000000 --- a/server/www/packages/packages-linux/x64/PIL/OleFileIO.py +++ /dev/null @@ -1,4 +0,0 @@ -raise ImportError( - 'PIL.OleFileIO is deprecated. Use the olefile Python package ' - 'instead. This module will be removed in a future version.' -) diff --git a/server/www/packages/packages-linux/x64/PIL/PSDraw.py b/server/www/packages/packages-linux/x64/PIL/PSDraw.py index d2ded6f..f37701c 100644 --- a/server/www/packages/packages-linux/x64/PIL/PSDraw.py +++ b/server/www/packages/packages-linux/x64/PIL/PSDraw.py @@ -15,9 +15,10 @@ # See the README file for information on usage and redistribution. # +import sys + from . import EpsImagePlugin from ._util import py3 -import sys ## # Simple Postscript graphics interface. @@ -38,16 +39,18 @@ class PSDraw(object): if not py3 or self.fp == sys.stdout: self.fp.write(to_write) else: - self.fp.write(bytes(to_write, 'UTF-8')) + self.fp.write(bytes(to_write, "UTF-8")) def begin_document(self, id=None): """Set up printing of a document. (Write Postscript DSC header.)""" # FIXME: incomplete - self._fp_write("%!PS-Adobe-3.0\n" - "save\n" - "/showpage { } def\n" - "%%EndComments\n" - "%%BeginDocument\n") + self._fp_write( + "%!PS-Adobe-3.0\n" + "save\n" + "/showpage { } def\n" + "%%EndComments\n" + "%%BeginDocument\n" + ) # self._fp_write(ERROR_PS) # debugging! self._fp_write(EDROFF_PS) self._fp_write(VDI_PS) @@ -56,9 +59,7 @@ class PSDraw(object): def end_document(self): """Ends printing. (Write Postscript DSC footer.)""" - self._fp_write("%%EndDocument\n" - "restore showpage\n" - "%%End\n") + self._fp_write("%%EndDocument\nrestore showpage\n%%End\n") if hasattr(self.fp, "flush"): self.fp.flush() @@ -71,8 +72,7 @@ class PSDraw(object): """ if font not in self.isofont: # reencode font - self._fp_write("/PSDraw-%s ISOLatin1Encoding /%s E\n" % - (font, font)) + self._fp_write("/PSDraw-%s ISOLatin1Encoding /%s E\n" % (font, font)) self.isofont[font] = 1 # rough self._fp_write("/F0 %d /PSDraw-%s F\n" % (size, font)) @@ -142,6 +142,7 @@ class PSDraw(object): EpsImagePlugin._save(im, self.fp, None, 0) self._fp_write("\ngrestore\n") + # -------------------------------------------------------------------- # Postscript driver diff --git a/server/www/packages/packages-linux/x64/PIL/PaletteFile.py b/server/www/packages/packages-linux/x64/PIL/PaletteFile.py index 9ed69d6..ab22d5f 100644 --- a/server/www/packages/packages-linux/x64/PIL/PaletteFile.py +++ b/server/www/packages/packages-linux/x64/PIL/PaletteFile.py @@ -15,10 +15,10 @@ from ._binary import o8 - ## # File handler for Teragon-style palette files. + class PaletteFile(object): rawmode = "RGB" diff --git a/server/www/packages/packages-linux/x64/PIL/PalmImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/PalmImagePlugin.py index 7d7b165..dd068d7 100644 --- a/server/www/packages/packages-linux/x64/PIL/PalmImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/PalmImagePlugin.py @@ -10,9 +10,12 @@ from . import Image, ImageFile from ._binary import o8, o16be as o16b +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "1.0" -_Palm8BitColormapValues = ( +# fmt: off +_Palm8BitColormapValues = ( # noqa: E131 (255, 255, 255), (255, 204, 255), (255, 153, 255), (255, 102, 255), (255, 51, 255), (255, 0, 255), (255, 255, 204), (255, 204, 204), (255, 153, 204), (255, 102, 204), (255, 51, 204), (255, 0, 204), @@ -77,6 +80,7 @@ _Palm8BitColormapValues = ( (0, 0, 0), (0, 0, 0), (0, 0, 0), (0, 0, 0), (0, 0, 0), (0, 0, 0), (0, 0, 0), (0, 0, 0), (0, 0, 0), (0, 0, 0), (0, 0, 0), (0, 0, 0)) +# fmt: on # so build a prototype image to be used for palette resampling @@ -86,7 +90,7 @@ def build_prototype_image(): palettedata = () for colormapValue in _Palm8BitColormapValues: palettedata += colormapValue - palettedata += (0, 0, 0)*(256 - len(_Palm8BitColormapValues)) + palettedata += (0, 0, 0) * (256 - len(_Palm8BitColormapValues)) image.putpalette(palettedata) return image @@ -98,17 +102,9 @@ Palm8BitColormapImage = build_prototype_image() # # -------------------------------------------------------------------- -_FLAGS = { - "custom-colormap": 0x4000, - "is-compressed": 0x8000, - "has-transparent": 0x2000, - } +_FLAGS = {"custom-colormap": 0x4000, "is-compressed": 0x8000, "has-transparent": 0x2000} -_COMPRESSION_TYPES = { - "none": 0xFF, - "rle": 0x01, - "scanline": 0x00, - } +_COMPRESSION_TYPES = {"none": 0xFF, "rle": 0x01, "scanline": 0x00} # @@ -117,6 +113,7 @@ _COMPRESSION_TYPES = { ## # (Internal) Image save plugin for the Palm format. + def _save(im, fp, filename): if im.mode == "P": @@ -128,28 +125,24 @@ def _save(im, fp, filename): bpp = 8 version = 1 - elif (im.mode == "L" and - "bpp" in im.encoderinfo and - im.encoderinfo["bpp"] in (1, 2, 4)): + elif im.mode == "L": + if im.encoderinfo.get("bpp") in (1, 2, 4): + # this is 8-bit grayscale, so we shift it to get the high-order bits, + # and invert it because + # Palm does greyscale from white (0) to black (1) + bpp = im.encoderinfo["bpp"] + im = im.point( + lambda x, shift=8 - bpp, maxval=(1 << bpp) - 1: maxval - (x >> shift) + ) + elif im.info.get("bpp") in (1, 2, 4): + # here we assume that even though the inherent mode is 8-bit grayscale, + # only the lower bpp bits are significant. + # We invert them to match the Palm. + bpp = im.info["bpp"] + im = im.point(lambda x, maxval=(1 << bpp) - 1: maxval - (x & maxval)) + else: + raise IOError("cannot write mode %s as Palm" % im.mode) - # this is 8-bit grayscale, so we shift it to get the high-order bits, - # and invert it because - # Palm does greyscale from white (0) to black (1) - bpp = im.encoderinfo["bpp"] - im = im.point( - lambda x, shift=8-bpp, maxval=(1 << bpp)-1: maxval - (x >> shift)) - # we ignore the palette here - im.mode = "P" - rawmode = "P;" + str(bpp) - version = 1 - - elif im.mode == "L" and "bpp" in im.info and im.info["bpp"] in (1, 2, 4): - - # here we assume that even though the inherent mode is 8-bit grayscale, - # only the lower bpp bits are significant. - # We invert them to match the Palm. - bpp = im.info["bpp"] - im = im.point(lambda x, maxval=(1 << bpp)-1: maxval - (x & maxval)) # we ignore the palette here im.mode = "P" rawmode = "P;" + str(bpp) @@ -175,7 +168,7 @@ def _save(im, fp, filename): cols = im.size[0] rows = im.size[1] - rowbytes = int((cols + (16//bpp - 1)) / (16 // bpp)) * 2 + rowbytes = int((cols + (16 // bpp - 1)) / (16 // bpp)) * 2 transparent_index = 0 compression_type = _COMPRESSION_TYPES["none"] @@ -199,7 +192,7 @@ def _save(im, fp, filename): fp.write(o16b(offset)) fp.write(o8(transparent_index)) fp.write(o8(compression_type)) - fp.write(o16b(0)) # reserved by Palm + fp.write(o16b(0)) # reserved by Palm # now write colormap if necessary @@ -207,20 +200,21 @@ def _save(im, fp, filename): fp.write(o16b(256)) for i in range(256): fp.write(o8(i)) - if colormapmode == 'RGB': + if colormapmode == "RGB": fp.write( - o8(colormap[3 * i]) + - o8(colormap[3 * i + 1]) + - o8(colormap[3 * i + 2])) - elif colormapmode == 'RGBA': + o8(colormap[3 * i]) + + o8(colormap[3 * i + 1]) + + o8(colormap[3 * i + 2]) + ) + elif colormapmode == "RGBA": fp.write( - o8(colormap[4 * i]) + - o8(colormap[4 * i + 1]) + - o8(colormap[4 * i + 2])) + o8(colormap[4 * i]) + + o8(colormap[4 * i + 1]) + + o8(colormap[4 * i + 2]) + ) # now convert data to raw form - ImageFile._save( - im, fp, [("raw", (0, 0)+im.size, 0, (rawmode, rowbytes, 1))]) + ImageFile._save(im, fp, [("raw", (0, 0) + im.size, 0, (rawmode, rowbytes, 1))]) if hasattr(fp, "flush"): fp.flush() diff --git a/server/www/packages/packages-linux/x64/PIL/PcdImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/PcdImagePlugin.py index 87e5792..6f01845 100644 --- a/server/www/packages/packages-linux/x64/PIL/PcdImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/PcdImagePlugin.py @@ -18,6 +18,8 @@ from . import Image, ImageFile from ._binary import i8 +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.1" @@ -26,6 +28,7 @@ __version__ = "0.1" # image from the file; higher resolutions are encoded in a proprietary # encoding. + class PcdImageFile(ImageFile.ImageFile): format = "PCD" @@ -49,7 +52,7 @@ class PcdImageFile(ImageFile.ImageFile): self.mode = "RGB" self._size = 768, 512 # FIXME: not correct for rotated images! - self.tile = [("pcd", (0, 0)+self.size, 96*2048, None)] + self.tile = [("pcd", (0, 0) + self.size, 96 * 2048, None)] def load_end(self): if self.tile_post_rotate: diff --git a/server/www/packages/packages-linux/x64/PIL/PcfFontFile.py b/server/www/packages/packages-linux/x64/PIL/PcfFontFile.py index eba85fe..0741246 100644 --- a/server/www/packages/packages-linux/x64/PIL/PcfFontFile.py +++ b/server/www/packages/packages-linux/x64/PIL/PcfFontFile.py @@ -16,39 +16,42 @@ # See the README file for information on usage and redistribution. # -from . import Image, FontFile -from ._binary import i8, i16le as l16, i32le as l32, i16be as b16, i32be as b32 +import io + +from . import FontFile, Image +from ._binary import i8, i16be as b16, i16le as l16, i32be as b32, i32le as l32 # -------------------------------------------------------------------- # declarations PCF_MAGIC = 0x70636601 # "\x01fcp" -PCF_PROPERTIES = (1 << 0) -PCF_ACCELERATORS = (1 << 1) -PCF_METRICS = (1 << 2) -PCF_BITMAPS = (1 << 3) -PCF_INK_METRICS = (1 << 4) -PCF_BDF_ENCODINGS = (1 << 5) -PCF_SWIDTHS = (1 << 6) -PCF_GLYPH_NAMES = (1 << 7) -PCF_BDF_ACCELERATORS = (1 << 8) +PCF_PROPERTIES = 1 << 0 +PCF_ACCELERATORS = 1 << 1 +PCF_METRICS = 1 << 2 +PCF_BITMAPS = 1 << 3 +PCF_INK_METRICS = 1 << 4 +PCF_BDF_ENCODINGS = 1 << 5 +PCF_SWIDTHS = 1 << 6 +PCF_GLYPH_NAMES = 1 << 7 +PCF_BDF_ACCELERATORS = 1 << 8 BYTES_PER_ROW = [ - lambda bits: ((bits+7) >> 3), - lambda bits: ((bits+15) >> 3) & ~1, - lambda bits: ((bits+31) >> 3) & ~3, - lambda bits: ((bits+63) >> 3) & ~7, + lambda bits: ((bits + 7) >> 3), + lambda bits: ((bits + 15) >> 3) & ~1, + lambda bits: ((bits + 31) >> 3) & ~3, + lambda bits: ((bits + 63) >> 3) & ~7, ] def sz(s, o): - return s[o:s.index(b"\0", o)] + return s[o : s.index(b"\0", o)] ## # Font file plugin for the X11 PCF format. + class PcfFontFile(FontFile.FontFile): name = "name" @@ -82,7 +85,7 @@ class PcfFontFile(FontFile.FontFile): ix = encoding[ch] if ix is not None: x, y, l, r, w, a, d, f = metrics[ix] - glyph = (w, 0), (l, d-y, x+l, d), (0, 0, x, y), bitmaps[ix] + glyph = (w, 0), (l, d - y, x + l, d), (0, 0, x, y), bitmaps[ix] self.glyph[ch] = glyph def _getformat(self, tag): @@ -117,7 +120,7 @@ class PcfFontFile(FontFile.FontFile): for i in range(nprops): p.append((i32(fp.read(4)), i8(fp.read(1)), i32(fp.read(4)))) if nprops & 3: - fp.seek(4 - (nprops & 3), 1) # pad + fp.seek(4 - (nprops & 3), io.SEEK_CUR) # pad data = fp.read(i32(fp.read(4))) @@ -140,7 +143,7 @@ class PcfFontFile(FontFile.FontFile): append = metrics.append - if (format & 0xff00) == 0x100: + if (format & 0xFF00) == 0x100: # "compressed" metrics for i in range(i16(fp.read(2))): @@ -151,10 +154,7 @@ class PcfFontFile(FontFile.FontFile): descent = i8(fp.read(1)) - 128 xsize = right - left ysize = ascent + descent - append( - (xsize, ysize, left, right, width, - ascent, descent, 0) - ) + append((xsize, ysize, left, right, width, ascent, descent, 0)) else: @@ -168,10 +168,7 @@ class PcfFontFile(FontFile.FontFile): attributes = i16(fp.read(2)) xsize = right - left ysize = ascent + descent - append( - (xsize, ysize, left, right, width, - ascent, descent, attributes) - ) + append((xsize, ysize, left, right, width, ascent, descent, attributes)) return metrics @@ -198,7 +195,7 @@ class PcfFontFile(FontFile.FontFile): bitmapSizes.append(i32(fp.read(4))) # byteorder = format & 4 # non-zero => MSB - bitorder = format & 8 # non-zero => MSB + bitorder = format & 8 # non-zero => MSB padindex = format & 3 bitmapsize = bitmapSizes[padindex] @@ -213,10 +210,8 @@ class PcfFontFile(FontFile.FontFile): for i in range(nbitmaps): x, y, l, r, w, a, d, f = metrics[i] - b, e = offsets[i], offsets[i+1] - bitmaps.append( - Image.frombytes("1", (x, y), data[b:e], "raw", mode, pad(x)) - ) + b, e = offsets[i], offsets[i + 1] + bitmaps.append(Image.frombytes("1", (x, y), data[b:e], "raw", mode, pad(x))) return bitmaps @@ -230,7 +225,7 @@ class PcfFontFile(FontFile.FontFile): firstCol, lastCol = i16(fp.read(2)), i16(fp.read(2)) firstRow, lastRow = i16(fp.read(2)), i16(fp.read(2)) - default = i16(fp.read(2)) + i16(fp.read(2)) # default nencoding = (lastCol - firstCol + 1) * (lastRow - firstRow + 1) @@ -238,7 +233,7 @@ class PcfFontFile(FontFile.FontFile): encodingOffset = i16(fp.read(2)) if encodingOffset != 0xFFFF: try: - encoding[i+firstCol] = encodingOffset + encoding[i + firstCol] = encodingOffset except IndexError: break # only load ISO-8859-1 glyphs diff --git a/server/www/packages/packages-linux/x64/PIL/PcxImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/PcxImagePlugin.py index daa58b3..397af8c 100644 --- a/server/www/packages/packages-linux/x64/PIL/PcxImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/PcxImagePlugin.py @@ -25,12 +25,16 @@ # See the README file for information on usage and redistribution. # +import io import logging + from . import Image, ImageFile, ImagePalette from ._binary import i8, i16le as i16, o8, o16le as o16 logger = logging.getLogger(__name__) +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.6" @@ -41,6 +45,7 @@ def _accept(prefix): ## # Image plugin for Paintbrush images. + class PcxImageFile(ImageFile.ImageFile): format = "PCX" @@ -54,7 +59,7 @@ class PcxImageFile(ImageFile.ImageFile): raise SyntaxError("not a PCX file") # image - bbox = i16(s, 4), i16(s, 6), i16(s, 8)+1, i16(s, 10)+1 + bbox = i16(s, 4), i16(s, 6), i16(s, 8) + 1, i16(s, 10) + 1 if bbox[2] <= bbox[0] or bbox[3] <= bbox[1]: raise SyntaxError("bad PCX image size") logger.debug("BBox: %s %s %s %s", *bbox) @@ -64,8 +69,13 @@ class PcxImageFile(ImageFile.ImageFile): bits = i8(s[3]) planes = i8(s[65]) stride = i16(s, 66) - logger.debug("PCX version %s, bits %s, planes %s, stride %s", - version, bits, planes, stride) + logger.debug( + "PCX version %s, bits %s, planes %s, stride %s", + version, + bits, + planes, + stride, + ) self.info["dpi"] = i16(s, 12), i16(s, 14) @@ -80,12 +90,12 @@ class PcxImageFile(ImageFile.ImageFile): elif version == 5 and bits == 8 and planes == 1: mode = rawmode = "L" # FIXME: hey, this doesn't work with the incremental loader !!! - self.fp.seek(-769, 2) + self.fp.seek(-769, io.SEEK_END) s = self.fp.read(769) if len(s) == 769 and i8(s[0]) == 12: # check if the palette is linear greyscale for i in range(256): - if s[i*3+1:i*3+4] != o8(i)*3: + if s[i * 3 + 1 : i * 3 + 4] != o8(i) * 3: mode = rawmode = "P" break if mode == "P": @@ -100,13 +110,14 @@ class PcxImageFile(ImageFile.ImageFile): raise IOError("unknown PCX mode") self.mode = mode - self._size = bbox[2]-bbox[0], bbox[3]-bbox[1] + self._size = bbox[2] - bbox[0], bbox[3] - bbox[1] bbox = (0, 0) + self.size logger.debug("size: %sx%s", *self.size) self.tile = [("pcx", bbox, self.fp.tell(), (rawmode, planes * stride))] + # -------------------------------------------------------------------- # save PCX files @@ -135,8 +146,12 @@ def _save(im, fp, filename): # Ideally it should be passed in in the state, but the bytes value # gets overwritten. - logger.debug("PcxImagePlugin._save: xwidth: %d, bits: %d, stride: %d", - im.size[0], bits, stride) + logger.debug( + "PcxImagePlugin._save: xwidth: %d, bits: %d, stride: %d", + im.size[0], + bits, + stride, + ) # under windows, we could determine the current screen size with # "Image.core.display_mode()[1]", but I think that's overkill... @@ -147,17 +162,30 @@ def _save(im, fp, filename): # PCX header fp.write( - o8(10) + o8(version) + o8(1) + o8(bits) + o16(0) + - o16(0) + o16(im.size[0]-1) + o16(im.size[1]-1) + o16(dpi[0]) + - o16(dpi[1]) + b"\0"*24 + b"\xFF"*24 + b"\0" + o8(planes) + - o16(stride) + o16(1) + o16(screen[0]) + o16(screen[1]) + - b"\0"*54 - ) + o8(10) + + o8(version) + + o8(1) + + o8(bits) + + o16(0) + + o16(0) + + o16(im.size[0] - 1) + + o16(im.size[1] - 1) + + o16(dpi[0]) + + o16(dpi[1]) + + b"\0" * 24 + + b"\xFF" * 24 + + b"\0" + + o8(planes) + + o16(stride) + + o16(1) + + o16(screen[0]) + + o16(screen[1]) + + b"\0" * 54 + ) assert fp.tell() == 128 - ImageFile._save(im, fp, [("pcx", (0, 0)+im.size, 0, - (rawmode, bits*planes))]) + ImageFile._save(im, fp, [("pcx", (0, 0) + im.size, 0, (rawmode, bits * planes))]) if im.mode == "P": # colour palette @@ -167,7 +195,8 @@ def _save(im, fp, filename): # greyscale palette fp.write(o8(12)) for i in range(256): - fp.write(o8(i)*3) + fp.write(o8(i) * 3) + # -------------------------------------------------------------------- # registry @@ -177,3 +206,5 @@ Image.register_open(PcxImageFile.format, PcxImageFile, _accept) Image.register_save(PcxImageFile.format, _save) Image.register_extension(PcxImageFile.format, ".pcx") + +Image.register_mime(PcxImageFile.format, "image/x-pcx") diff --git a/server/www/packages/packages-linux/x64/PIL/PdfImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/PdfImagePlugin.py index b425027..1fd40f5 100644 --- a/server/www/packages/packages-linux/x64/PIL/PdfImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/PdfImagePlugin.py @@ -20,11 +20,14 @@ # Image plugin for PDF images (output only). ## -from . import Image, ImageFile, ImageSequence, PdfParser import io import os import time +from . import Image, ImageFile, ImageSequence, PdfParser + +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.5" @@ -46,6 +49,7 @@ def _save_all(im, fp, filename): ## # (Internal) Image save plugin for the PDF format. + def _save(im, fp, filename, save_all=False): is_appending = im.encoderinfo.get("append", False) if is_appending: @@ -56,16 +60,16 @@ def _save(im, fp, filename, save_all=False): resolution = im.encoderinfo.get("resolution", 72.0) info = { - "title": None if is_appending else os.path.splitext( - os.path.basename(filename) - )[0], + "title": None + if is_appending + else os.path.splitext(os.path.basename(filename))[0], "author": None, "subject": None, "keywords": None, "creator": None, "producer": None, "creationDate": None if is_appending else time.gmtime(), - "modDate": None if is_appending else time.gmtime() + "modDate": None if is_appending else time.gmtime(), } for k, default in info.items(): v = im.encoderinfo.get(k) if k in im.encoderinfo else default @@ -140,7 +144,7 @@ def _save(im, fp, filename, save_all=False): PdfParser.PdfName("Indexed"), PdfParser.PdfName("DeviceRGB"), 255, - PdfParser.PdfBinary(palette) + PdfParser.PdfBinary(palette), ] procset = "ImageI" # indexed color elif im.mode == "RGB": @@ -164,16 +168,15 @@ def _save(im, fp, filename, save_all=False): # FIXME: the hex encoder doesn't support packed 1-bit # images; do things the hard way... data = im.tobytes("raw", "1") - im = Image.new("L", (len(data), 1), None) + im = Image.new("L", im.size) im.putdata(data) - ImageFile._save(im, op, [("hex", (0, 0)+im.size, 0, im.mode)]) + ImageFile._save(im, op, [("hex", (0, 0) + im.size, 0, im.mode)]) elif filter == "DCTDecode": Image.SAVE["JPEG"](im, op, filename) elif filter == "FlateDecode": - ImageFile._save(im, op, [("zip", (0, 0)+im.size, 0, im.mode)]) + ImageFile._save(im, op, [("zip", (0, 0) + im.size, 0, im.mode)]) elif filter == "RunLengthDecode": - ImageFile._save(im, op, - [("packbits", (0, 0)+im.size, 0, im.mode)]) + ImageFile._save(im, op, [("packbits", (0, 0) + im.size, 0, im.mode)]) else: raise ValueError("unsupported PDF filter (%s)" % filter) @@ -182,48 +185,46 @@ def _save(im, fp, filename, save_all=False): width, height = im.size - existing_pdf.write_obj(image_refs[pageNumber], - stream=op.getvalue(), - Type=PdfParser.PdfName("XObject"), - Subtype=PdfParser.PdfName("Image"), - Width=width, # * 72.0 / resolution, - Height=height, # * 72.0 / resolution, - Filter=PdfParser.PdfName(filter), - BitsPerComponent=bits, - DecodeParams=params, - ColorSpace=colorspace) + existing_pdf.write_obj( + image_refs[pageNumber], + stream=op.getvalue(), + Type=PdfParser.PdfName("XObject"), + Subtype=PdfParser.PdfName("Image"), + Width=width, # * 72.0 / resolution, + Height=height, # * 72.0 / resolution, + Filter=PdfParser.PdfName(filter), + BitsPerComponent=bits, + DecodeParams=params, + ColorSpace=colorspace, + ) # # page - existing_pdf.write_page(page_refs[pageNumber], - Resources=PdfParser.PdfDict( - ProcSet=[ - PdfParser.PdfName("PDF"), - PdfParser.PdfName(procset) - ], - XObject=PdfParser.PdfDict( - image=image_refs[pageNumber] - ) - ), - MediaBox=[ - 0, - 0, - int(width * 72.0 / resolution), - int(height * 72.0 / resolution) - ], - Contents=contents_refs[pageNumber]) + existing_pdf.write_page( + page_refs[pageNumber], + Resources=PdfParser.PdfDict( + ProcSet=[PdfParser.PdfName("PDF"), PdfParser.PdfName(procset)], + XObject=PdfParser.PdfDict(image=image_refs[pageNumber]), + ), + MediaBox=[ + 0, + 0, + int(width * 72.0 / resolution), + int(height * 72.0 / resolution), + ], + Contents=contents_refs[pageNumber], + ) # # page contents page_contents = PdfParser.make_bytes( - "q %d 0 0 %d 0 0 cm /image Do Q\n" % ( - int(width * 72.0 / resolution), - int(height * 72.0 / resolution))) + "q %d 0 0 %d 0 0 cm /image Do Q\n" + % (int(width * 72.0 / resolution), int(height * 72.0 / resolution)) + ) - existing_pdf.write_obj(contents_refs[pageNumber], - stream=page_contents) + existing_pdf.write_obj(contents_refs[pageNumber], stream=page_contents) pageNumber += 1 @@ -234,6 +235,7 @@ def _save(im, fp, filename, save_all=False): fp.flush() existing_pdf.close() + # # -------------------------------------------------------------------- diff --git a/server/www/packages/packages-linux/x64/PIL/PdfParser.py b/server/www/packages/packages-linux/x64/PIL/PdfParser.py index bcd29db..0ec6bba 100644 --- a/server/www/packages/packages-linux/x64/PIL/PdfParser.py +++ b/server/www/packages/packages-linux/x64/PIL/PdfParser.py @@ -6,6 +6,7 @@ import os import re import time import zlib + from ._util import py3 try: @@ -15,11 +16,15 @@ except ImportError: if py3: # Python 3.x + def make_bytes(s): return s.encode("us-ascii") + + else: # Python 2.x + def make_bytes(s): # pragma: no cover - return s # pragma: no cover + return s # pragma: no cover # see 7.9.2.2 Text String Type on page 86 and D.3 PDFDocEncoding Character Set @@ -74,8 +79,8 @@ PDFDocEncoding = { def decode_text(b): - if b[:len(codecs.BOM_UTF16_BE)] == codecs.BOM_UTF16_BE: - return b[len(codecs.BOM_UTF16_BE):].decode("utf_16_be") + if b[: len(codecs.BOM_UTF16_BE)] == codecs.BOM_UTF16_BE: + return b[len(codecs.BOM_UTF16_BE) :].decode("utf_16_be") elif py3: # Python 3.x return "".join(PDFDocEncoding.get(byte, chr(byte)) for byte in b) else: # Python 2.x @@ -85,6 +90,7 @@ def decode_text(b): class PdfFormatError(RuntimeError): """An error that probably indicates a syntactic or semantic error in the PDF file structure""" + pass @@ -93,8 +99,9 @@ def check_format_condition(condition, error_message): raise PdfFormatError(error_message) -class IndirectReference(collections.namedtuple("IndirectReferenceTuple", - ["object_id", "generation"])): +class IndirectReference( + collections.namedtuple("IndirectReferenceTuple", ["object_id", "generation"]) +): def __str__(self): return "%s %s R" % self @@ -102,9 +109,11 @@ class IndirectReference(collections.namedtuple("IndirectReferenceTuple", return self.__str__().encode("us-ascii") def __eq__(self, other): - return other.__class__ is self.__class__ and \ - other.object_id == self.object_id and \ - other.generation == self.generation + return ( + other.__class__ is self.__class__ + and other.object_id == self.object_id + and other.generation == self.generation + ) def __ne__(self, other): return not (self == other) @@ -120,9 +129,9 @@ class IndirectObjectDef(IndirectReference): class XrefTable: def __init__(self): - self.existing_entries = {} # object ID => (offset, generation) - self.new_entries = {} # object ID => (offset, generation) - self.deleted_entries = {0: 65536} # object ID => generation + self.existing_entries = {} # object ID => (offset, generation) + self.new_entries = {} # object ID => (offset, generation) + self.deleted_entries = {0: 65536} # object ID => generation self.reading_finished = False def __setitem__(self, key, value): @@ -150,26 +159,27 @@ class XrefTable: elif key in self.deleted_entries: generation = self.deleted_entries[key] else: - raise IndexError("object ID " + str(key) + - " cannot be deleted because it doesn't exist") + raise IndexError( + "object ID " + str(key) + " cannot be deleted because it doesn't exist" + ) def __contains__(self, key): return key in self.existing_entries or key in self.new_entries def __len__(self): - return len(set(self.existing_entries.keys()) | - set(self.new_entries.keys()) | - set(self.deleted_entries.keys())) + return len( + set(self.existing_entries.keys()) + | set(self.new_entries.keys()) + | set(self.deleted_entries.keys()) + ) def keys(self): return ( - set(self.existing_entries.keys()) - - set(self.deleted_entries.keys()) + set(self.existing_entries.keys()) - set(self.deleted_entries.keys()) ) | set(self.new_entries.keys()) def write(self, f): - keys = sorted(set(self.new_entries.keys()) | - set(self.deleted_entries.keys())) + keys = sorted(set(self.new_entries.keys()) | set(self.deleted_entries.keys())) deleted_keys = sorted(set(self.deleted_entries.keys())) startxref = f.tell() f.write(b"xref\n") @@ -177,7 +187,7 @@ class XrefTable: # find a contiguous sequence of object IDs prev = None for index, key in enumerate(keys): - if prev is None or prev+1 == key: + if prev is None or prev + 1 == key: prev = key else: contiguous_keys = keys[:index] @@ -186,25 +196,27 @@ class XrefTable: else: contiguous_keys = keys keys = None - f.write(make_bytes("%d %d\n" % - (contiguous_keys[0], len(contiguous_keys)))) + f.write(make_bytes("%d %d\n" % (contiguous_keys[0], len(contiguous_keys)))) for object_id in contiguous_keys: if object_id in self.new_entries: - f.write(make_bytes("%010d %05d n \n" % - self.new_entries[object_id])) + f.write(make_bytes("%010d %05d n \n" % self.new_entries[object_id])) else: this_deleted_object_id = deleted_keys.pop(0) - check_format_condition(object_id == this_deleted_object_id, - "expected the next deleted object " - "ID to be %s, instead found %s" % - (object_id, this_deleted_object_id)) + check_format_condition( + object_id == this_deleted_object_id, + "expected the next deleted object ID to be %s, instead found %s" + % (object_id, this_deleted_object_id), + ) try: next_in_linked_list = deleted_keys[0] except IndexError: next_in_linked_list = 0 - f.write(make_bytes("%010d %05d f \n" % - (next_in_linked_list, - self.deleted_entries[object_id]))) + f.write( + make_bytes( + "%010d %05d f \n" + % (next_in_linked_list, self.deleted_entries[object_id]) + ) + ) return startxref @@ -221,8 +233,9 @@ class PdfName: return self.name.decode("us-ascii") def __eq__(self, other): - return (isinstance(other, PdfName) and other.name == self.name) or \ - other == self.name + return ( + isinstance(other, PdfName) and other.name == self.name + ) or other == self.name def __hash__(self): return hash(self.name) @@ -269,36 +282,31 @@ class PdfDict(UserDict): else: self.__dict__[key] = value else: - if isinstance(key, str): - key = key.encode("us-ascii") - self[key] = value + self[key.encode("us-ascii")] = value def __getattr__(self, key): try: - value = self[key] + value = self[key.encode("us-ascii")] except KeyError: - try: - value = self[key.encode("us-ascii")] - except KeyError: - raise AttributeError(key) + raise AttributeError(key) if isinstance(value, bytes): value = decode_text(value) if key.endswith("Date"): if value.startswith("D:"): value = value[2:] - relationship = 'Z' + relationship = "Z" if len(value) > 17: relationship = value[14] offset = int(value[15:17]) * 60 if len(value) > 20: offset += int(value[18:20]) - format = '%Y%m%d%H%M%S'[:len(value) - 2] - value = time.strptime(value[:len(format)+2], format) - if relationship in ['+', '-']: + format = "%Y%m%d%H%M%S"[: len(value) - 2] + value = time.strptime(value[: len(format) + 2], format) + if relationship in ["+", "-"]: offset *= 60 - if relationship == '+': + if relationship == "+": offset *= -1 value = time.gmtime(calendar.timegm(value) + offset) return value @@ -325,9 +333,12 @@ class PdfBinary: self.data = data if py3: # Python 3.x + def __bytes__(self): return make_bytes("<%s>" % "".join("%02X" % b for b in self.data)) + else: # Python 2.x + def __str__(self): return "<%s>" % "".join("%02X" % ord(b) for b in self.data) @@ -350,8 +361,8 @@ class PdfStream: return zlib.decompress(self.buf, bufsize=int(expected_length)) else: raise NotImplementedError( - "stream filter %s unknown/unsupported" % - repr(self.dictionary.Filter)) + "stream filter %s unknown/unsupported" % repr(self.dictionary.Filter) + ) def pdf_repr(x): @@ -361,19 +372,19 @@ def pdf_repr(x): return b"false" elif x is None: return b"null" - elif (isinstance(x, PdfName) or isinstance(x, PdfDict) or - isinstance(x, PdfArray) or isinstance(x, PdfBinary)): + elif isinstance(x, (PdfName, PdfDict, PdfArray, PdfBinary)): return bytes(x) elif isinstance(x, int): return str(x).encode("us-ascii") elif isinstance(x, time.struct_time): - return b'(D:'+time.strftime('%Y%m%d%H%M%SZ', x).encode("us-ascii")+b')' + return b"(D:" + time.strftime("%Y%m%d%H%M%SZ", x).encode("us-ascii") + b")" elif isinstance(x, dict): return bytes(PdfDict(x)) elif isinstance(x, list): return bytes(PdfArray(x)) - elif ((py3 and isinstance(x, str)) or - (not py3 and isinstance(x, unicode))): + elif (py3 and isinstance(x, str)) or ( + not py3 and isinstance(x, unicode) # noqa: F821 + ): return pdf_repr(encode_text(x)) elif isinstance(x, bytes): # XXX escape more chars? handle binary garbage @@ -386,17 +397,14 @@ def pdf_repr(x): class PdfParser: - """Based on https://www.adobe.com/content/dam/acom/en/devnet/acrobat/pdfs/PDF32000_2008.pdf + """Based on + https://www.adobe.com/content/dam/acom/en/devnet/acrobat/pdfs/PDF32000_2008.pdf Supports PDF up to 1.4 """ - def __init__(self, filename=None, f=None, - buf=None, start_offset=0, mode="rb"): - # type: (PdfParser, str, file, Union[bytes, bytearray], int, str) - # -> None + def __init__(self, filename=None, f=None, buf=None, start_offset=0, mode="rb"): if buf and f: - raise RuntimeError( - "specify buf or f or filename, but not both buf and f") + raise RuntimeError("specify buf or f or filename, but not both buf and f") self.filename = filename self.buf = buf self.f = f @@ -470,13 +478,13 @@ class PdfParser: self.root_ref = self.next_object_id(self.f.tell()) self.pages_ref = self.next_object_id(0) self.rewrite_pages() - self.write_obj(self.root_ref, - Type=PdfName(b"Catalog"), - Pages=self.pages_ref) - self.write_obj(self.pages_ref, - Type=PdfName(b"Pages"), - Count=len(self.pages), - Kids=self.pages) + self.write_obj(self.root_ref, Type=PdfName(b"Catalog"), Pages=self.pages_ref) + self.write_obj( + self.pages_ref, + Type=PdfName(b"Pages"), + Count=len(self.pages), + Kids=self.pages, + ) return self.root_ref def rewrite_pages(self): @@ -522,8 +530,11 @@ class PdfParser: if self.info: trailer_dict[b"Info"] = self.info_ref self.last_xref_section_offset = start_xref - self.f.write(b"trailer\n" + bytes(PdfDict(trailer_dict)) + - make_bytes("\nstartxref\n%d\n%%%%EOF" % start_xref)) + self.f.write( + b"trailer\n" + + bytes(PdfDict(trailer_dict)) + + make_bytes("\nstartxref\n%d\n%%%%EOF" % start_xref) + ) def write_page(self, ref, *objs, **dict_obj): if isinstance(ref, int): @@ -585,12 +596,14 @@ class PdfParser: else: self.info = PdfDict(self.read_indirect(self.info_ref)) check_format_condition(b"Type" in self.root, "/Type missing in Root") - check_format_condition(self.root[b"Type"] == b"Catalog", - "/Type in Root is not /Catalog") + check_format_condition( + self.root[b"Type"] == b"Catalog", "/Type in Root is not /Catalog" + ) check_format_condition(b"Pages" in self.root, "/Pages missing in Root") - check_format_condition(isinstance(self.root[b"Pages"], - IndirectReference), - "/Pages in Root is not an indirect reference") + check_format_condition( + isinstance(self.root[b"Pages"], IndirectReference), + "/Pages in Root is not an indirect reference", + ) self.pages_ref = self.root[b"Pages"] self.page_tree_root = self.read_indirect(self.pages_ref) self.pages = self.linearize_page_tree(self.page_tree_root) @@ -618,13 +631,34 @@ class PdfParser: newline_only = br"[\r\n]+" newline = whitespace_optional + newline_only + whitespace_optional re_trailer_end = re.compile( - whitespace_mandatory + br"trailer" + whitespace_optional + - br"\<\<(.*\>\>)" + newline + br"startxref" + newline + br"([0-9]+)" + - newline + br"%%EOF" + whitespace_optional + br"$", re.DOTALL) + whitespace_mandatory + + br"trailer" + + whitespace_optional + + br"\<\<(.*\>\>)" + + newline + + br"startxref" + + newline + + br"([0-9]+)" + + newline + + br"%%EOF" + + whitespace_optional + + br"$", + re.DOTALL, + ) re_trailer_prev = re.compile( - whitespace_optional + br"trailer" + whitespace_optional + - br"\<\<(.*?\>\>)" + newline + br"startxref" + newline + br"([0-9]+)" + - newline + br"%%EOF" + whitespace_optional, re.DOTALL) + whitespace_optional + + br"trailer" + + whitespace_optional + + br"\<\<(.*?\>\>)" + + newline + + br"startxref" + + newline + + br"([0-9]+)" + + newline + + br"%%EOF" + + whitespace_optional, + re.DOTALL, + ) def read_trailer(self): search_start_offset = len(self.buf) - 16384 @@ -636,7 +670,7 @@ class PdfParser: last_match = m while m: last_match = m - m = self.re_trailer_end.search(self.buf, m.start()+16) + m = self.re_trailer_end.search(self.buf, m.start() + 16) if not m: m = last_match trailer_data = m.group(1) @@ -648,26 +682,29 @@ class PdfParser: self.read_prev_trailer(self.trailer_dict[b"Prev"]) def read_prev_trailer(self, xref_section_offset): - trailer_offset = self.read_xref_table( - xref_section_offset=xref_section_offset) + trailer_offset = self.read_xref_table(xref_section_offset=xref_section_offset) m = self.re_trailer_prev.search( - self.buf[trailer_offset:trailer_offset+16384]) + self.buf[trailer_offset : trailer_offset + 16384] + ) check_format_condition(m, "previous trailer not found") trailer_data = m.group(1) - check_format_condition(int(m.group(2)) == xref_section_offset, - "xref section offset in previous trailer " - "doesn't match what was expected") + check_format_condition( + int(m.group(2)) == xref_section_offset, + "xref section offset in previous trailer doesn't match what was expected", + ) trailer_dict = self.interpret_trailer(trailer_data) if b"Prev" in trailer_dict: self.read_prev_trailer(trailer_dict[b"Prev"]) re_whitespace_optional = re.compile(whitespace_optional) re_name = re.compile( - whitespace_optional + br"/([!-$&'*-.0-;=?-Z\\^-z|~]+)(?=" + - delimiter_or_ws + br")") + whitespace_optional + + br"/([!-$&'*-.0-;=?-Z\\^-z|~]+)(?=" + + delimiter_or_ws + + br")" + ) re_dict_start = re.compile(whitespace_optional + br"\<\<") - re_dict_end = re.compile( - whitespace_optional + br"\>\>" + whitespace_optional) + re_dict_end = re.compile(whitespace_optional + br"\>\>" + whitespace_optional) @classmethod def interpret_trailer(cls, trailer_data): @@ -679,19 +716,21 @@ class PdfParser: m = cls.re_dict_end.match(trailer_data, offset) check_format_condition( m and m.end() == len(trailer_data), - "name not found in trailer, remaining data: " + - repr(trailer_data[offset:])) + "name not found in trailer, remaining data: " + + repr(trailer_data[offset:]), + ) break key = cls.interpret_name(m.group(1)) value, offset = cls.get_value(trailer_data, m.end()) trailer[key] = value check_format_condition( b"Size" in trailer and isinstance(trailer[b"Size"], int), - "/Size not in trailer or not an integer") + "/Size not in trailer or not an integer", + ) check_format_condition( - b"Root" in trailer and - isinstance(trailer[b"Root"], IndirectReference), - "/Root not in trailer or not an indirect reference") + b"Root" in trailer and isinstance(trailer[b"Root"], IndirectReference), + "/Root not in trailer or not an indirect reference", + ) return trailer re_hashes_in_name = re.compile(br"([^#]*)(#([0-9a-fA-F]{2}))?") @@ -701,8 +740,7 @@ class PdfParser: name = b"" for m in cls.re_hashes_in_name.finditer(raw): if m.group(3): - name += m.group(1) + \ - bytearray.fromhex(m.group(3).decode("us-ascii")) + name += m.group(1) + bytearray.fromhex(m.group(3).decode("us-ascii")) else: name += m.group(1) if as_text: @@ -710,37 +748,54 @@ class PdfParser: else: return bytes(name) - re_null = re.compile( - whitespace_optional + br"null(?=" + delimiter_or_ws + br")") - re_true = re.compile( - whitespace_optional + br"true(?=" + delimiter_or_ws + br")") - re_false = re.compile( - whitespace_optional + br"false(?=" + delimiter_or_ws + br")") + re_null = re.compile(whitespace_optional + br"null(?=" + delimiter_or_ws + br")") + re_true = re.compile(whitespace_optional + br"true(?=" + delimiter_or_ws + br")") + re_false = re.compile(whitespace_optional + br"false(?=" + delimiter_or_ws + br")") re_int = re.compile( - whitespace_optional + br"([-+]?[0-9]+)(?=" + delimiter_or_ws + br")") + whitespace_optional + br"([-+]?[0-9]+)(?=" + delimiter_or_ws + br")" + ) re_real = re.compile( - whitespace_optional + br"([-+]?([0-9]+\.[0-9]*|[0-9]*\.[0-9]+))(?=" + - delimiter_or_ws + br")") + whitespace_optional + + br"([-+]?([0-9]+\.[0-9]*|[0-9]*\.[0-9]+))(?=" + + delimiter_or_ws + + br")" + ) re_array_start = re.compile(whitespace_optional + br"\[") re_array_end = re.compile(whitespace_optional + br"]") re_string_hex = re.compile( - whitespace_optional + br"\<(" + whitespace_or_hex + br"*)\>") + whitespace_optional + br"\<(" + whitespace_or_hex + br"*)\>" + ) re_string_lit = re.compile(whitespace_optional + br"\(") re_indirect_reference = re.compile( - whitespace_optional + br"([-+]?[0-9]+)" + whitespace_mandatory + - br"([-+]?[0-9]+)" + whitespace_mandatory + br"R(?=" + delimiter_or_ws + - br")") + whitespace_optional + + br"([-+]?[0-9]+)" + + whitespace_mandatory + + br"([-+]?[0-9]+)" + + whitespace_mandatory + + br"R(?=" + + delimiter_or_ws + + br")" + ) re_indirect_def_start = re.compile( - whitespace_optional + br"([-+]?[0-9]+)" + whitespace_mandatory + - br"([-+]?[0-9]+)" + whitespace_mandatory + br"obj(?=" + - delimiter_or_ws + br")") + whitespace_optional + + br"([-+]?[0-9]+)" + + whitespace_mandatory + + br"([-+]?[0-9]+)" + + whitespace_mandatory + + br"obj(?=" + + delimiter_or_ws + + br")" + ) re_indirect_def_end = re.compile( - whitespace_optional + br"endobj(?=" + delimiter_or_ws + br")") + whitespace_optional + br"endobj(?=" + delimiter_or_ws + br")" + ) re_comment = re.compile( - br"(" + whitespace_optional + br"%[^\r\n]*" + newline + br")*") + br"(" + whitespace_optional + br"%[^\r\n]*" + newline + br")*" + ) re_stream_start = re.compile(whitespace_optional + br"stream\r?\n") re_stream_end = re.compile( - whitespace_optional + br"endstream(?=" + delimiter_or_ws + br")") + whitespace_optional + br"endstream(?=" + delimiter_or_ws + br")" + ) @classmethod def get_value(cls, data, offset, expect_indirect=None, max_nesting=-1): @@ -753,32 +808,37 @@ class PdfParser: if m: check_format_condition( int(m.group(1)) > 0, - "indirect object definition: object ID must be greater than 0") + "indirect object definition: object ID must be greater than 0", + ) check_format_condition( int(m.group(2)) >= 0, - "indirect object definition: generation must be non-negative") + "indirect object definition: generation must be non-negative", + ) check_format_condition( - expect_indirect is None or expect_indirect == - IndirectReference(int(m.group(1)), int(m.group(2))), - "indirect object definition different than expected") - object, offset = cls.get_value( - data, m.end(), max_nesting=max_nesting-1) + expect_indirect is None + or expect_indirect + == IndirectReference(int(m.group(1)), int(m.group(2))), + "indirect object definition different than expected", + ) + object, offset = cls.get_value(data, m.end(), max_nesting=max_nesting - 1) if offset is None: return object, None m = cls.re_indirect_def_end.match(data, offset) - check_format_condition( - m, "indirect object definition end not found") + check_format_condition(m, "indirect object definition end not found") return object, m.end() check_format_condition( - not expect_indirect, "indirect object definition not found") + not expect_indirect, "indirect object definition not found" + ) m = cls.re_indirect_reference.match(data, offset) if m: check_format_condition( int(m.group(1)) > 0, - "indirect object reference: object ID must be greater than 0") + "indirect object reference: object ID must be greater than 0", + ) check_format_condition( int(m.group(2)) >= 0, - "indirect object reference: generation must be non-negative") + "indirect object reference: generation must be non-negative", + ) return IndirectReference(int(m.group(1)), int(m.group(2))), m.end() m = cls.re_dict_start.match(data, offset) if m: @@ -786,12 +846,10 @@ class PdfParser: result = {} m = cls.re_dict_end.match(data, offset) while not m: - key, offset = cls.get_value( - data, offset, max_nesting=max_nesting-1) + key, offset = cls.get_value(data, offset, max_nesting=max_nesting - 1) if offset is None: return result, None - value, offset = cls.get_value( - data, offset, max_nesting=max_nesting-1) + value, offset = cls.get_value(data, offset, max_nesting=max_nesting - 1) result[key] = value if offset is None: return result, None @@ -803,9 +861,10 @@ class PdfParser: stream_len = int(result[b"Length"]) except (TypeError, KeyError, ValueError): raise PdfFormatError( - "bad or missing Length in stream dict (%r)" % - result.get(b"Length", None)) - stream_data = data[m.end():m.end() + stream_len] + "bad or missing Length in stream dict (%r)" + % result.get(b"Length", None) + ) + stream_data = data[m.end() : m.end() + stream_len] m = cls.re_stream_end.match(data, m.end() + stream_len) check_format_condition(m, "stream end not found") offset = m.end() @@ -819,8 +878,7 @@ class PdfParser: result = [] m = cls.re_array_end.match(data, offset) while not m: - value, offset = cls.get_value( - data, offset, max_nesting=max_nesting-1) + value, offset = cls.get_value(data, offset, max_nesting=max_nesting - 1) result.append(value) if offset is None: return result, None @@ -848,10 +906,9 @@ class PdfParser: m = cls.re_string_hex.match(data, offset) if m: # filter out whitespace - hex_string = bytearray([ - b for b in m.group(1) - if b in b"0123456789abcdefABCDEF" - ]) + hex_string = bytearray( + [b for b in m.group(1) if b in b"0123456789abcdefABCDEF"] + ) if len(hex_string) % 2 == 1: # append a 0 if the length is not even - yes, at the end hex_string.append(ord(b"0")) @@ -860,10 +917,11 @@ class PdfParser: if m: return cls.get_literal_string(data, m.end()) # return None, offset # fallback (only for debugging) - raise PdfFormatError( - "unrecognized object: " + repr(data[offset:offset+32])) + raise PdfFormatError("unrecognized object: " + repr(data[offset : offset + 32])) - re_lit_str_token = re.compile(br"(\\[nrtbf()\\])|(\\[0-9]{1,3})|(\\(\r\n|\r|\n))|(\r\n|\r|\n)|(\()|(\))") + re_lit_str_token = re.compile( + br"(\\[nrtbf()\\])|(\\[0-9]{1,3})|(\\(\r\n|\r|\n))|(\r\n|\r|\n)|(\()|(\))" + ) escaped_chars = { b"n": b"\n", b"r": b"\r", @@ -881,14 +939,14 @@ class PdfParser: ord(b"("): b"(", ord(b")"): b")", ord(b"\\"): b"\\", - } + } @classmethod def get_literal_string(cls, data, offset): nesting_depth = 0 result = bytearray() for m in cls.re_lit_str_token.finditer(data, offset): - result.extend(data[offset:m.start()]) + result.extend(data[offset : m.start()]) if m.group(1): result.extend(cls.escaped_chars[m.group(1)[1]]) elif m.group(2): @@ -908,30 +966,36 @@ class PdfParser: offset = m.end() raise PdfFormatError("unfinished literal string") - re_xref_section_start = re.compile( - whitespace_optional + br"xref" + newline) + re_xref_section_start = re.compile(whitespace_optional + br"xref" + newline) re_xref_subsection_start = re.compile( - whitespace_optional + br"([0-9]+)" + whitespace_mandatory + - br"([0-9]+)" + whitespace_optional + newline_only) + whitespace_optional + + br"([0-9]+)" + + whitespace_mandatory + + br"([0-9]+)" + + whitespace_optional + + newline_only + ) re_xref_entry = re.compile(br"([0-9]{10}) ([0-9]{5}) ([fn])( \r| \n|\r\n)") def read_xref_table(self, xref_section_offset): subsection_found = False m = self.re_xref_section_start.match( - self.buf, xref_section_offset + self.start_offset) + self.buf, xref_section_offset + self.start_offset + ) check_format_condition(m, "xref section start not found") offset = m.end() while True: m = self.re_xref_subsection_start.match(self.buf, offset) if not m: check_format_condition( - subsection_found, "xref subsection start not found") + subsection_found, "xref subsection start not found" + ) break subsection_found = True offset = m.end() first_object = int(m.group(1)) num_objects = int(m.group(2)) - for i in range(first_object, first_object+num_objects): + for i in range(first_object, first_object + num_objects): m = self.re_xref_entry.match(self.buf, offset) check_format_condition(m, "xref entry not found") offset = m.end() @@ -940,9 +1004,9 @@ class PdfParser: if not is_free: new_entry = (int(m.group(1)), generation) check_format_condition( - i not in self.xref_table or - self.xref_table[i] == new_entry, - "xref entry duplicated (and not identical)") + i not in self.xref_table or self.xref_table[i] == new_entry, + "xref entry duplicated (and not identical)", + ) self.xref_table[i] = new_entry return offset @@ -952,10 +1016,14 @@ class PdfParser: generation == ref[1], "expected to find generation %s for object ID %s in xref table, " "instead found generation %s at offset %s" - % (ref[1], ref[0], generation, offset)) - value = self.get_value(self.buf, offset + self.start_offset, - expect_indirect=IndirectReference(*ref), - max_nesting=max_nesting)[0] + % (ref[1], ref[0], generation, offset), + ) + value = self.get_value( + self.buf, + offset + self.start_offset, + expect_indirect=IndirectReference(*ref), + max_nesting=max_nesting, + )[0] self.cached_objects[ref] = value return value @@ -963,7 +1031,8 @@ class PdfParser: if node is None: node = self.page_tree_root check_format_condition( - node[b"Type"] == b"Pages", "/Type of page tree node is not /Pages") + node[b"Type"] == b"Pages", "/Type of page tree node is not /Pages" + ) pages = [] for kid in node[b"Kids"]: kid_object = self.read_indirect(kid) diff --git a/server/www/packages/packages-linux/x64/PIL/PixarImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/PixarImagePlugin.py index d07b28d..dc71ca1 100644 --- a/server/www/packages/packages-linux/x64/PIL/PixarImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/PixarImagePlugin.py @@ -22,12 +22,15 @@ from . import Image, ImageFile from ._binary import i16le as i16 +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.1" # # helpers + def _accept(prefix): return prefix[:4] == b"\200\350\000\000" @@ -35,6 +38,7 @@ def _accept(prefix): ## # Image plugin for PIXAR raster images. + class PixarImageFile(ImageFile.ImageFile): format = "PIXAR" @@ -60,7 +64,7 @@ class PixarImageFile(ImageFile.ImageFile): # FIXME: to be continued... # create tile descriptor (assuming "dumped") - self.tile = [("raw", (0, 0)+self.size, 1024, (self.mode, 0, 1))] + self.tile = [("raw", (0, 0) + self.size, 1024, (self.mode, 0, 1))] # diff --git a/server/www/packages/packages-linux/x64/PIL/PngImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/PngImagePlugin.py index 15077fc..be237b3 100644 --- a/server/www/packages/packages-linux/x64/PIL/PngImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/PngImagePlugin.py @@ -33,13 +33,15 @@ import logging import re -import zlib import struct +import zlib from . import Image, ImageFile, ImagePalette from ._binary import i8, i16be as i16, i32be as i32, o16be as o16, o32be as o32 from ._util import py3 +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.9" logger = logging.getLogger(__name__) @@ -52,25 +54,30 @@ _MAGIC = b"\211PNG\r\n\032\n" _MODES = { # supported bits/color combinations, and corresponding modes/rawmodes - (1, 0): ("1", "1"), - (2, 0): ("L", "L;2"), - (4, 0): ("L", "L;4"), - (8, 0): ("L", "L"), + # Greyscale + (1, 0): ("1", "1"), + (2, 0): ("L", "L;2"), + (4, 0): ("L", "L;4"), + (8, 0): ("L", "L"), (16, 0): ("I", "I;16B"), - (8, 2): ("RGB", "RGB"), + # Truecolour + (8, 2): ("RGB", "RGB"), (16, 2): ("RGB", "RGB;16B"), - (1, 3): ("P", "P;1"), - (2, 3): ("P", "P;2"), - (4, 3): ("P", "P;4"), - (8, 3): ("P", "P"), - (8, 4): ("LA", "LA"), + # Indexed-colour + (1, 3): ("P", "P;1"), + (2, 3): ("P", "P;2"), + (4, 3): ("P", "P;4"), + (8, 3): ("P", "P"), + # Greyscale with alpha + (8, 4): ("LA", "LA"), (16, 4): ("RGBA", "LA;16B"), # LA;16B->LA not yet available - (8, 6): ("RGBA", "RGBA"), + # Truecolour with alpha + (8, 6): ("RGBA", "RGBA"), (16, 6): ("RGBA", "RGBA;16B"), } -_simple_palette = re.compile(b'^\xff*\x00\xff*$') +_simple_palette = re.compile(b"^\xff*\x00\xff*$") # Maximum decompressed size for a iTXt or zTXt chunk. # Eliminates decompression bombs where compressed chunks can expand 1000x @@ -88,21 +95,21 @@ def _safe_zlib_decompress(s): def _crc32(data, seed=0): - return zlib.crc32(data, seed) & 0xffffffff + return zlib.crc32(data, seed) & 0xFFFFFFFF # -------------------------------------------------------------------- # Support classes. Suitable for PNG and related formats like MNG etc. -class ChunkStream(object): +class ChunkStream(object): def __init__(self, fp): self.fp = fp self.queue = [] def read(self): - "Fetch a new chunk. Returns header information." + """Fetch a new chunk. Returns header information.""" cid = None if self.queue: @@ -134,13 +141,13 @@ class ChunkStream(object): self.queue.append((cid, pos, length)) def call(self, cid, pos, length): - "Call the appropriate chunk handler" + """Call the appropriate chunk handler""" logger.debug("STREAM %r %s %s", cid, pos, length) - return getattr(self, "chunk_" + cid.decode('ascii'))(pos, length) + return getattr(self, "chunk_" + cid.decode("ascii"))(pos, length) def crc(self, cid, data): - "Read and verify checksum" + """Read and verify checksum""" # Skip CRC checks for ancillary chunks if allowed to load truncated # images @@ -153,14 +160,12 @@ class ChunkStream(object): crc1 = _crc32(data, _crc32(cid)) crc2 = i32(self.fp.read(4)) if crc1 != crc2: - raise SyntaxError("broken PNG file (bad header checksum in %r)" - % cid) + raise SyntaxError("broken PNG file (bad header checksum in %r)" % cid) except struct.error: - raise SyntaxError("broken PNG file (incomplete checksum in %r)" - % cid) + raise SyntaxError("broken PNG file (incomplete checksum in %r)" % cid) def crc_skip(self, cid, data): - "Read checksum. Used if the C module is not present" + """Read checksum. Used if the C module is not present""" self.fp.read(4) @@ -191,8 +196,9 @@ class iTXt(str): keeping their extra information """ + @staticmethod - def __new__(cls, text, lang, tkey): + def __new__(cls, text, lang=None, tkey=None): """ :param cls: the class to use when creating the instance :param text: value for this key @@ -246,11 +252,12 @@ class PngInfo(object): tkey = tkey.encode("utf-8", "strict") if zip: - self.add(b"iTXt", key + b"\0\x01\0" + lang + b"\0" + tkey + b"\0" + - zlib.compress(value)) + self.add( + b"iTXt", + key + b"\0\x01\0" + lang + b"\0" + tkey + b"\0" + zlib.compress(value), + ) else: - self.add(b"iTXt", key + b"\0\0\0" + lang + b"\0" + tkey + b"\0" + - value) + self.add(b"iTXt", key + b"\0\0\0" + lang + b"\0" + tkey + b"\0" + value) def add_text(self, key, value, zip=False): """Appends a text chunk. @@ -267,12 +274,12 @@ class PngInfo(object): # The tEXt chunk stores latin-1 text if not isinstance(value, bytes): try: - value = value.encode('latin-1', 'strict') + value = value.encode("latin-1", "strict") except UnicodeError: return self.add_itxt(key, value, zip=zip) if not isinstance(key, bytes): - key = key.encode('latin-1', 'strict') + key = key.encode("latin-1", "strict") if zip: self.add(b"zTXt", key + b"\0\0" + zlib.compress(value)) @@ -283,8 +290,8 @@ class PngInfo(object): # -------------------------------------------------------------------- # PNG image stream (IHDR/IEND) -class PngStream(ChunkStream): +class PngStream(ChunkStream): def __init__(self, fp): ChunkStream.__init__(self, fp) @@ -296,14 +303,17 @@ class PngStream(ChunkStream): self.im_mode = None self.im_tile = None self.im_palette = None + self.im_custom_mimetype = None self.text_memory = 0 def check_text_memory(self, chunklen): self.text_memory += chunklen if self.text_memory > MAX_TEXT_MEMORY: - raise ValueError("Too much memory used in text chunks: " - "%s>MAX_TEXT_MEMORY" % self.text_memory) + raise ValueError( + "Too much memory used in text chunks: %s>MAX_TEXT_MEMORY" + % self.text_memory + ) def chunk_iCCP(self, pos, length): @@ -319,10 +329,11 @@ class PngStream(ChunkStream): logger.debug("Compression method %s", i8(s[i])) comp_method = i8(s[i]) if comp_method != 0: - raise SyntaxError("Unknown compression method %s in iCCP chunk" % - comp_method) + raise SyntaxError( + "Unknown compression method %s in iCCP chunk" % comp_method + ) try: - icc_profile = _safe_zlib_decompress(s[i+2:]) + icc_profile = _safe_zlib_decompress(s[i + 2 :]) except ValueError: if ImageFile.LOAD_TRUNCATED_IMAGES: icc_profile = None @@ -340,7 +351,7 @@ class PngStream(ChunkStream): self.im_size = i32(s), i32(s[4:]) try: self.im_mode, self.im_rawmode = _MODES[(i8(s[8]), i8(s[9]))] - except: + except Exception: pass if i8(s[12]): self.im_info["interlace"] = 1 @@ -351,7 +362,7 @@ class PngStream(ChunkStream): def chunk_IDAT(self, pos, length): # image data - self.im_tile = [("zip", (0, 0)+self.im_size, pos, self.im_rawmode)] + self.im_tile = [("zip", (0, 0) + self.im_size, pos, self.im_rawmode)] self.im_idat = length raise EOFError @@ -383,7 +394,7 @@ class PngStream(ChunkStream): # otherwise, we have a byte string with one alpha value # for each palette entry self.im_info["transparency"] = s - elif self.im_mode == "L": + elif self.im_mode in ("1", "L", "I"): self.im_info["transparency"] = i16(s) elif self.im_mode == "RGB": self.im_info["transparency"] = i16(s), i16(s[2:]), i16(s[4:]) @@ -400,8 +411,8 @@ class PngStream(ChunkStream): # WP x,y, Red x,y, Green x,y Blue x,y s = ImageFile._safe_read(self.fp, length) - raw_vals = struct.unpack('>%dI' % (len(s) // 4), s) - self.im_info['chromaticity'] = tuple(elt/100000.0 for elt in raw_vals) + raw_vals = struct.unpack(">%dI" % (len(s) // 4), s) + self.im_info["chromaticity"] = tuple(elt / 100000.0 for elt in raw_vals) return s def chunk_sRGB(self, pos, length): @@ -412,7 +423,7 @@ class PngStream(ChunkStream): # 3 absolute colorimetric s = ImageFile._safe_read(self.fp, length) - self.im_info['srgb'] = i8(s) + self.im_info["srgb"] = i8(s) return s def chunk_pHYs(self, pos, length): @@ -440,8 +451,8 @@ class PngStream(ChunkStream): v = b"" if k: if py3: - k = k.decode('latin-1', 'strict') - v = v.decode('latin-1', 'replace') + k = k.decode("latin-1", "strict") + v = v.decode("latin-1", "replace") self.im_info[k] = self.im_text[k] = v self.check_text_memory(len(v)) @@ -462,8 +473,9 @@ class PngStream(ChunkStream): else: comp_method = 0 if comp_method != 0: - raise SyntaxError("Unknown compression method %s in zTXt chunk" % - comp_method) + raise SyntaxError( + "Unknown compression method %s in zTXt chunk" % comp_method + ) try: v = _safe_zlib_decompress(v[1:]) except ValueError: @@ -476,8 +488,8 @@ class PngStream(ChunkStream): if k: if py3: - k = k.decode('latin-1', 'strict') - v = v.decode('latin-1', 'replace') + k = k.decode("latin-1", "strict") + v = v.decode("latin-1", "replace") self.im_info[k] = self.im_text[k] = v self.check_text_memory(len(v)) @@ -526,10 +538,22 @@ class PngStream(ChunkStream): return s + def chunk_eXIf(self, pos, length): + s = ImageFile._safe_read(self.fp, length) + self.im_info["exif"] = b"Exif\x00\x00" + s + return s + + # APNG chunks + def chunk_acTL(self, pos, length): + s = ImageFile._safe_read(self.fp, length) + self.im_custom_mimetype = "image/apng" + return s + # -------------------------------------------------------------------- # PNG reader + def _accept(prefix): return prefix[:8] == _MAGIC @@ -537,6 +561,7 @@ def _accept(prefix): ## # Image plugin for PNG images. + class PngImageFile(ImageFile.ImageFile): format = "PNG" @@ -579,17 +604,27 @@ class PngImageFile(ImageFile.ImageFile): self.mode = self.png.im_mode self._size = self.png.im_size self.info = self.png.im_info - self.text = self.png.im_text # experimental + self._text = None self.tile = self.png.im_tile + self.custom_mimetype = self.png.im_custom_mimetype if self.png.im_palette: rawmode, data = self.png.im_palette self.palette = ImagePalette.raw(rawmode, data) - self.__idat = length # used by load_read() + self.__prepare_idat = length # used by load_prepare() + + @property + def text(self): + # experimental + if self._text is None: + # iTxt, tEXt and zTXt chunks may appear at the end of the file + # So load the file to ensure that they are read + self.load() + return self._text def verify(self): - "Verify PNG file" + """Verify PNG file""" if self.fp is None: raise RuntimeError("verify must be called directly after open") @@ -600,18 +635,21 @@ class PngImageFile(ImageFile.ImageFile): self.png.verify() self.png.close() + if self._exclusive_fp: + self.fp.close() self.fp = None def load_prepare(self): - "internal: prepare to read PNG file" + """internal: prepare to read PNG file""" if self.info.get("interlace"): self.decoderconfig = self.decoderconfig + (1,) + self.__idat = self.__prepare_idat # used by load_read() ImageFile.ImageFile.load_prepare(self) def load_read(self, read_bytes): - "internal: read more image data" + """internal: read more image data""" while self.__idat == 0: # end of chunk, skip forward to next one @@ -637,30 +675,63 @@ class PngImageFile(ImageFile.ImageFile): return self.fp.read(read_bytes) def load_end(self): - "internal: finished reading image data" + """internal: finished reading image data""" + while True: + self.fp.read(4) # CRC + try: + cid, pos, length = self.png.read() + except (struct.error, SyntaxError): + break + + if cid == b"IEND": + break + + try: + self.png.call(cid, pos, length) + except UnicodeDecodeError: + break + except EOFError: + ImageFile._safe_read(self.fp, length) + except AttributeError: + logger.debug("%r %s %s (unknown)", cid, pos, length) + ImageFile._safe_read(self.fp, length) + self._text = self.png.im_text self.png.close() self.png = None + def _getexif(self): + if "exif" not in self.info: + self.load() + if "exif" not in self.info: + return None + return dict(self.getexif()) + + def getexif(self): + if "exif" not in self.info: + self.load() + return ImageFile.ImageFile.getexif(self) + # -------------------------------------------------------------------- # PNG writer _OUTMODES = { # supported PIL modes, and corresponding rawmodes/bits/color combinations - "1": ("1", b'\x01\x00'), - "L;1": ("L;1", b'\x01\x00'), - "L;2": ("L;2", b'\x02\x00'), - "L;4": ("L;4", b'\x04\x00'), - "L": ("L", b'\x08\x00'), - "LA": ("LA", b'\x08\x04'), - "I": ("I;16B", b'\x10\x00'), - "P;1": ("P;1", b'\x01\x03'), - "P;2": ("P;2", b'\x02\x03'), - "P;4": ("P;4", b'\x04\x03'), - "P": ("P", b'\x08\x03'), - "RGB": ("RGB", b'\x08\x02'), - "RGBA": ("RGBA", b'\x08\x06'), + "1": ("1", b"\x01\x00"), + "L;1": ("L;1", b"\x01\x00"), + "L;2": ("L;2", b"\x02\x00"), + "L;4": ("L;4", b"\x04\x00"), + "L": ("L", b"\x08\x00"), + "LA": ("LA", b"\x08\x04"), + "I": ("I;16B", b"\x10\x00"), + "I;16": ("I;16B", b"\x10\x00"), + "P;1": ("P;1", b"\x01\x03"), + "P;2": ("P;2", b"\x02\x03"), + "P;4": ("P;4", b"\x04\x03"), + "P": ("P", b"\x08\x03"), + "RGB": ("RGB", b"\x08\x02"), + "RGBA": ("RGBA", b"\x08\x06"), } @@ -701,7 +772,7 @@ def _save(im, fp, filename, chunk=putchunk): else: # check palette contents if im.palette: - colors = max(min(len(im.palette.getdata()[1])//3, 256), 2) + colors = max(min(len(im.palette.getdata()[1]) // 3, 256), 2) else: colors = 256 @@ -717,10 +788,12 @@ def _save(im, fp, filename, chunk=putchunk): mode = "%s;%d" % (mode, bits) # encoder options - im.encoderconfig = (im.encoderinfo.get("optimize", False), - im.encoderinfo.get("compress_level", -1), - im.encoderinfo.get("compress_type", -1), - im.encoderinfo.get("dictionary", b"")) + im.encoderconfig = ( + im.encoderinfo.get("optimize", False), + im.encoderinfo.get("compress_level", -1), + im.encoderinfo.get("compress_type", -1), + im.encoderinfo.get("dictionary", b""), + ) # get the corresponding PNG mode try: @@ -733,12 +806,16 @@ def _save(im, fp, filename, chunk=putchunk): fp.write(_MAGIC) - chunk(fp, b"IHDR", - o32(im.size[0]), o32(im.size[1]), # 0: size - mode, # 8: depth/type - b'\0', # 10: compression - b'\0', # 11: filter category - b'\0') # 12: interlace flag + chunk( + fp, + b"IHDR", + o32(im.size[0]), # 0: size + o32(im.size[1]), + mode, # 8: depth/type + b"\0", # 10: compression + b"\0", # 11: filter category + b"\0", # 12: interlace flag + ) chunks = [b"cHRM", b"gAMA", b"sBIT", b"sRGB", b"tIME"] @@ -772,23 +849,22 @@ def _save(im, fp, filename, chunk=putchunk): palette_byte_number = (2 ** bits) * 3 palette_bytes = im.im.getpalette("RGB")[:palette_byte_number] while len(palette_bytes) < palette_byte_number: - palette_bytes += b'\0' + palette_bytes += b"\0" chunk(fp, b"PLTE", palette_bytes) - transparency = im.encoderinfo.get('transparency', - im.info.get('transparency', None)) + transparency = im.encoderinfo.get("transparency", im.info.get("transparency", None)) if transparency or transparency == 0: if im.mode == "P": # limit to actual palette size - alpha_bytes = 2**bits + alpha_bytes = 2 ** bits if isinstance(transparency, bytes): chunk(fp, b"tRNS", transparency[:alpha_bytes]) else: transparency = max(0, min(255, transparency)) - alpha = b'\xFF' * transparency + b'\0' + alpha = b"\xFF" * transparency + b"\0" chunk(fp, b"tRNS", alpha[:alpha_bytes]) - elif im.mode == "L": + elif im.mode in ("1", "L", "I"): transparency = max(0, min(65535, transparency)) chunk(fp, b"tRNS", o16(transparency)) elif im.mode == "RGB": @@ -802,15 +878,18 @@ def _save(im, fp, filename, chunk=putchunk): else: if im.mode == "P" and im.im.getpalettemode() == "RGBA": alpha = im.im.getpalette("RGBA", "A") - alpha_bytes = 2**bits + alpha_bytes = 2 ** bits chunk(fp, b"tRNS", alpha[:alpha_bytes]) dpi = im.encoderinfo.get("dpi") if dpi: - chunk(fp, b"pHYs", - o32(int(dpi[0] / 0.0254 + 0.5)), - o32(int(dpi[1] / 0.0254 + 0.5)), - b'\x01') + chunk( + fp, + b"pHYs", + o32(int(dpi[0] / 0.0254 + 0.5)), + o32(int(dpi[1] / 0.0254 + 0.5)), + b"\x01", + ) info = im.encoderinfo.get("pnginfo") if info: @@ -820,8 +899,15 @@ def _save(im, fp, filename, chunk=putchunk): chunks.remove(cid) chunk(fp, cid, data) - ImageFile._save(im, _idat(fp, chunk), - [("zip", (0, 0)+im.size, 0, rawmode)]) + exif = im.encoderinfo.get("exif", im.info.get("exif")) + if exif: + if isinstance(exif, Image.Exif): + exif = exif.tobytes(8) + if exif.startswith(b"Exif\x00\x00"): + exif = exif[6:] + chunk(fp, b"eXIf", exif) + + ImageFile._save(im, _idat(fp, chunk), [("zip", (0, 0) + im.size, 0, rawmode)]) chunk(fp, b"IEND", b"") @@ -832,6 +918,7 @@ def _save(im, fp, filename, chunk=putchunk): # -------------------------------------------------------------------- # PNG chunk converter + def getchunks(im, **params): """Return a list of PNG chunks representing this image.""" @@ -866,6 +953,6 @@ def getchunks(im, **params): Image.register_open(PngImageFile.format, PngImageFile, _accept) Image.register_save(PngImageFile.format, _save) -Image.register_extension(PngImageFile.format, ".png") +Image.register_extensions(PngImageFile.format, [".png", ".apng"]) Image.register_mime(PngImageFile.format, "image/png") diff --git a/server/www/packages/packages-linux/x64/PIL/PpmImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/PpmImagePlugin.py index 8002678..c3e9eed 100644 --- a/server/www/packages/packages-linux/x64/PIL/PpmImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/PpmImagePlugin.py @@ -17,12 +17,14 @@ from . import Image, ImageFile +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.2" # # -------------------------------------------------------------------- -b_whitespace = b'\x20\x09\x0a\x0b\x0c\x0d' +b_whitespace = b"\x20\x09\x0a\x0b\x0c\x0d" MODES = { # standard @@ -34,7 +36,7 @@ MODES = { # PIL extensions (for test purposes only) b"PyP": "P", b"PyRGBA": "RGBA", - b"PyCMYK": "CMYK" + b"PyCMYK": "CMYK", } @@ -45,6 +47,7 @@ def _accept(prefix): ## # Image plugin for PBM, PGM, and PPM images. + class PpmImageFile(ImageFile.ImageFile): format = "PPM" @@ -55,10 +58,10 @@ class PpmImageFile(ImageFile.ImageFile): c = self.fp.read(1) if not c or c in b_whitespace: break - if c > b'\x79': + if c > b"\x79": raise ValueError("Expected ASCII value, found binary") s = s + c - if (len(s) > 9): + if len(s) > 9: raise ValueError("Expected int, got > 9 digits") return s @@ -68,7 +71,14 @@ class PpmImageFile(ImageFile.ImageFile): s = self.fp.read(1) if s != b"P": raise SyntaxError("not a PPM file") - mode = MODES[self._token(s)] + magic_number = self._token(s) + mode = MODES[magic_number] + + self.custom_mimetype = { + b"P4": "image/x-portable-bitmap", + b"P5": "image/x-portable-graymap", + b"P6": "image/x-portable-pixmap", + }.get(magic_number) if mode == "1": self.mode = "1" @@ -83,8 +93,7 @@ class PpmImageFile(ImageFile.ImageFile): if s not in b_whitespace: break if s == b"": - raise ValueError( - "File does not extend beyond magic number") + raise ValueError("File does not extend beyond magic number") if s != b"#": break s = self.fp.readline() @@ -98,32 +107,30 @@ class PpmImageFile(ImageFile.ImageFile): elif ix == 2: # maxgrey if s > 255: - if not mode == 'L': + if not mode == "L": raise ValueError("Too many colors for band: %s" % s) - if s < 2**16: - self.mode = 'I' - rawmode = 'I;16B' + if s < 2 ** 16: + self.mode = "I" + rawmode = "I;16B" else: - self.mode = 'I' - rawmode = 'I;32B' + self.mode = "I" + rawmode = "I;32B" self._size = xsize, ysize - self.tile = [("raw", - (0, 0, xsize, ysize), - self.fp.tell(), - (rawmode, 0, 1))] + self.tile = [("raw", (0, 0, xsize, ysize), self.fp.tell(), (rawmode, 0, 1))] # # -------------------------------------------------------------------- + def _save(im, fp, filename): if im.mode == "1": rawmode, head = "1;I", b"P4" elif im.mode == "L": rawmode, head = "L", b"P5" elif im.mode == "I": - if im.getextrema()[1] < 2**16: + if im.getextrema()[1] < 2 ** 16: rawmode, head = "I;16B", b"P5" else: rawmode, head = "I;32B", b"P5" @@ -133,7 +140,7 @@ def _save(im, fp, filename): rawmode, head = "RGB", b"P6" else: raise IOError("cannot write mode %s as PPM" % im.mode) - fp.write(head + ("\n%d %d\n" % im.size).encode('ascii')) + fp.write(head + ("\n%d %d\n" % im.size).encode("ascii")) if head == b"P6": fp.write(b"255\n") if head == b"P5": @@ -143,11 +150,12 @@ def _save(im, fp, filename): fp.write(b"65535\n") elif rawmode == "I;32B": fp.write(b"2147483648\n") - ImageFile._save(im, fp, [("raw", (0, 0)+im.size, 0, (rawmode, 0, 1))]) + ImageFile._save(im, fp, [("raw", (0, 0) + im.size, 0, (rawmode, 0, 1))]) # ALTERNATIVE: save via builtin debug function # im._dump(filename) + # # -------------------------------------------------------------------- @@ -155,4 +163,6 @@ def _save(im, fp, filename): Image.register_open(PpmImageFile.format, PpmImageFile, _accept) Image.register_save(PpmImageFile.format, _save) -Image.register_extensions(PpmImageFile.format, [".pbm", ".pgm", ".ppm"]) +Image.register_extensions(PpmImageFile.format, [".pbm", ".pgm", ".ppm", ".pnm"]) + +Image.register_mime(PpmImageFile.format, "image/x-portable-anymap") diff --git a/server/www/packages/packages-linux/x64/PIL/PsdImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/PsdImagePlugin.py index 2d64ecd..f72ad5f 100644 --- a/server/www/packages/packages-linux/x64/PIL/PsdImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/PsdImagePlugin.py @@ -16,8 +16,12 @@ # See the README file for information on usage and redistribution. # +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.4" +import io + from . import Image, ImageFile, ImagePalette from ._binary import i8, i16be as i16, i32be as i32 @@ -31,13 +35,14 @@ MODES = { (4, 8): ("CMYK", 4), (7, 8): ("L", 1), # FIXME: multilayer (8, 8): ("L", 1), # duotone - (9, 8): ("LAB", 3) + (9, 8): ("LAB", 3), } # --------------------------------------------------------------------. # read PSD images + def _accept(prefix): return prefix[:4] == b"8BPS" @@ -45,10 +50,12 @@ def _accept(prefix): ## # Image plugin for Photoshop images. + class PsdImageFile(ImageFile.ImageFile): format = "PSD" format_description = "Adobe Photoshop" + _close_exclusive_fp_after_loading = False def _open(self): @@ -92,13 +99,13 @@ class PsdImageFile(ImageFile.ImageFile): # load resources end = self.fp.tell() + size while self.fp.tell() < end: - signature = read(4) + read(4) # signature id = i16(read(2)) name = read(i8(read(1))) if not (len(name) & 1): read(1) # padding data = read(i32(read(4))) - if (len(data) & 1): + if len(data) & 1: read(1) # padding self.resources.append((id, name, data)) if id == 1039: # ICC profile @@ -123,7 +130,7 @@ class PsdImageFile(ImageFile.ImageFile): self.tile = _maketile(self.fp, mode, (0, 0) + self.size, channels) # keep the file open - self._fp = self.fp + self.__fp = self.fp self.frame = 1 self._min_frame = 1 @@ -141,11 +148,11 @@ class PsdImageFile(ImageFile.ImageFile): # seek to given layer (1..max) try: - name, mode, bbox, tile = self.layers[layer-1] + name, mode, bbox, tile = self.layers[layer - 1] self.mode = mode self.tile = tile self.frame = layer - self.fp = self._fp + self.fp = self.__fp return name, bbox except IndexError: raise EOFError("no such layer") @@ -156,13 +163,21 @@ class PsdImageFile(ImageFile.ImageFile): def load_prepare(self): # create image memory if necessary - if not self.im or\ - self.im.mode != self.mode or self.im.size != self.size: + if not self.im or self.im.mode != self.mode or self.im.size != self.size: self.im = Image.core.fill(self.mode, self.size, 0) # create palette (optional) if self.mode == "P": Image.Image.load(self) + def _close__fp(self): + try: + if self.__fp != self.fp: + self.__fp.close() + except AttributeError: + pass + finally: + self.__fp = None + def _layerinfo(file): # read layerinfo block @@ -207,33 +222,31 @@ def _layerinfo(file): mode = None # unknown # skip over blend flags and extra information - filler = read(12) + read(12) # filler name = "" - size = i32(read(4)) + size = i32(read(4)) # length of the extra data field combined = 0 if size: + data_end = file.tell() + size + length = i32(read(4)) if length: - mask_y = i32(read(4)) - mask_x = i32(read(4)) - mask_h = i32(read(4)) - mask_y - mask_w = i32(read(4)) - mask_x - file.seek(length - 16, 1) + file.seek(length - 16, io.SEEK_CUR) combined += length + 4 length = i32(read(4)) if length: - file.seek(length, 1) + file.seek(length, io.SEEK_CUR) combined += length + 4 length = i8(read(1)) if length: # Don't know the proper encoding, # Latin-1 should be a good guess - name = read(length).decode('latin-1', 'replace') + name = read(length).decode("latin-1", "replace") combined += length + 1 - file.seek(size - combined, 1) + file.seek(data_end) layers.append((name, mode, (x0, y0, x1, y1))) # get tiles @@ -271,7 +284,7 @@ def _maketile(file, mode, bbox, channels): if mode == "CMYK": layer += ";I" tile.append(("raw", bbox, offset, layer)) - offset = offset + xsize*ysize + offset = offset + xsize * ysize elif compression == 1: # @@ -284,11 +297,9 @@ def _maketile(file, mode, bbox, channels): layer = mode[channel] if mode == "CMYK": layer += ";I" - tile.append( - ("packbits", bbox, offset, layer) - ) + tile.append(("packbits", bbox, offset, layer)) for y in range(ysize): - offset = offset + i16(bytecount[i:i+2]) + offset = offset + i16(bytecount[i : i + 2]) i += 2 file.seek(offset) @@ -298,6 +309,7 @@ def _maketile(file, mode, bbox, channels): return tile + # -------------------------------------------------------------------- # registry diff --git a/server/www/packages/packages-linux/x64/PIL/PyAccess.py b/server/www/packages/packages-linux/x64/PIL/PyAccess.py index cce2de2..2ab06f9 100644 --- a/server/www/packages/packages-linux/x64/PIL/PyAccess.py +++ b/server/www/packages/packages-linux/x64/PIL/PyAccess.py @@ -25,7 +25,6 @@ import sys from cffi import FFI - logger = logging.getLogger(__name__) @@ -42,17 +41,18 @@ ffi.cdef(defs) class PyAccess(object): - def __init__(self, img, readonly=False): vals = dict(img.im.unsafe_ptrs) self.readonly = readonly - self.image8 = ffi.cast('unsigned char **', vals['image8']) - self.image32 = ffi.cast('int **', vals['image32']) - self.image = ffi.cast('unsigned char **', vals['image']) + self.image8 = ffi.cast("unsigned char **", vals["image8"]) + self.image32 = ffi.cast("int **", vals["image32"]) + self.image = ffi.cast("unsigned char **", vals["image"]) self.xsize, self.ysize = img.im.size # Keep pointer to im object to prevent dereferencing. self._im = img.im + if self._im.mode == "P": + self._palette = img.palette # Debugging is polluting test traces, only useful here # when hacking on PyAccess @@ -73,8 +73,22 @@ class PyAccess(object): :param color: The pixel value. """ if self.readonly: - raise ValueError('Attempt to putpixel a read only image') - (x, y) = self.check_xy(xy) + raise ValueError("Attempt to putpixel a read only image") + (x, y) = xy + if x < 0: + x = self.xsize + x + if y < 0: + y = self.ysize + y + (x, y) = self.check_xy((x, y)) + + if ( + self._im.mode == "P" + and isinstance(color, (list, tuple)) + and len(color) in [3, 4] + ): + # RGB or RGBA value for a P image + color = self._palette.getcolor(color) + return self.set_pixel(x, y, color) def __getitem__(self, xy): @@ -88,8 +102,12 @@ class PyAccess(object): :returns: a pixel value for single band images, a tuple of pixel values for multiband images. """ - - (x, y) = self.check_xy(xy) + (x, y) = xy + if x < 0: + x = self.xsize + x + if y < 0: + y = self.ysize + y + (x, y) = self.check_xy((x, y)) return self.get_pixel(x, y) putpixel = __setitem__ @@ -98,12 +116,13 @@ class PyAccess(object): def check_xy(self, xy): (x, y) = xy if not (0 <= x < self.xsize and 0 <= y < self.ysize): - raise ValueError('pixel location out of range') + raise ValueError("pixel location out of range") return xy class _PyAccess32_2(PyAccess): """ PA, LA, stored in first and last bytes of a 32 bit word """ + def _post_init(self, *args, **kwargs): self.pixels = ffi.cast("struct Pixel_RGBA **", self.image32) @@ -139,6 +158,7 @@ class _PyAccess32_3(PyAccess): class _PyAccess32_4(PyAccess): """ RGBA etc, all 4 bytes of a 32 bit word """ + def _post_init(self, *args, **kwargs): self.pixels = ffi.cast("struct Pixel_RGBA **", self.image32) @@ -157,6 +177,7 @@ class _PyAccess32_4(PyAccess): class _PyAccess8(PyAccess): """ 1, L, P, 8 bit images stored as uint8 """ + def _post_init(self, *args, **kwargs): self.pixels = self.image8 @@ -174,8 +195,9 @@ class _PyAccess8(PyAccess): class _PyAccessI16_N(PyAccess): """ I;16 access, native bitendian without conversion """ + def _post_init(self, *args, **kwargs): - self.pixels = ffi.cast('unsigned short **', self.image) + self.pixels = ffi.cast("unsigned short **", self.image) def get_pixel(self, x, y): return self.pixels[y][x] @@ -191,8 +213,9 @@ class _PyAccessI16_N(PyAccess): class _PyAccessI16_L(PyAccess): """ I;16L access, with conversion """ + def _post_init(self, *args, **kwargs): - self.pixels = ffi.cast('struct Pixel_I16 **', self.image) + self.pixels = ffi.cast("struct Pixel_I16 **", self.image) def get_pixel(self, x, y): pixel = self.pixels[y][x] @@ -205,14 +228,15 @@ class _PyAccessI16_L(PyAccess): except TypeError: color = min(color[0], 65535) - pixel.l = color & 0xFF + pixel.l = color & 0xFF # noqa: E741 pixel.r = color >> 8 class _PyAccessI16_B(PyAccess): """ I;16B access, with conversion """ + def _post_init(self, *args, **kwargs): - self.pixels = ffi.cast('struct Pixel_I16 **', self.image) + self.pixels = ffi.cast("struct Pixel_I16 **", self.image) def get_pixel(self, x, y): pixel = self.pixels[y][x] @@ -222,15 +246,16 @@ class _PyAccessI16_B(PyAccess): pixel = self.pixels[y][x] try: color = min(color, 65535) - except: + except Exception: color = min(color[0], 65535) - pixel.l = color >> 8 + pixel.l = color >> 8 # noqa: E741 pixel.r = color & 0xFF class _PyAccessI32_N(PyAccess): """ Signed Int32 access, native endian """ + def _post_init(self, *args, **kwargs): self.pixels = self.image32 @@ -243,15 +268,15 @@ class _PyAccessI32_N(PyAccess): class _PyAccessI32_Swap(PyAccess): """ I;32L/B access, with byteswapping conversion """ + def _post_init(self, *args, **kwargs): self.pixels = self.image32 def reverse(self, i): - orig = ffi.new('int *', i) - chars = ffi.cast('unsigned char *', orig) - chars[0], chars[1], chars[2], chars[3] = chars[3], chars[2], \ - chars[1], chars[0] - return ffi.cast('int *', chars)[0] + orig = ffi.new("int *", i) + chars = ffi.cast("unsigned char *", orig) + chars[0], chars[1], chars[2], chars[3] = chars[3], chars[2], chars[1], chars[0] + return ffi.cast("int *", chars)[0] def get_pixel(self, x, y): return self.reverse(self.pixels[y][x]) @@ -262,8 +287,9 @@ class _PyAccessI32_Swap(PyAccess): class _PyAccessF(PyAccess): """ 32 bit float access """ + def _post_init(self, *args, **kwargs): - self.pixels = ffi.cast('float **', self.image32) + self.pixels = ffi.cast("float **", self.image32) def get_pixel(self, x, y): return self.pixels[y][x] @@ -277,38 +303,39 @@ class _PyAccessF(PyAccess): self.pixels[y][x] = color[0] -mode_map = {'1': _PyAccess8, - 'L': _PyAccess8, - 'P': _PyAccess8, - 'LA': _PyAccess32_2, - 'La': _PyAccess32_2, - 'PA': _PyAccess32_2, - 'RGB': _PyAccess32_3, - 'LAB': _PyAccess32_3, - 'HSV': _PyAccess32_3, - 'YCbCr': _PyAccess32_3, - 'RGBA': _PyAccess32_4, - 'RGBa': _PyAccess32_4, - 'RGBX': _PyAccess32_4, - 'CMYK': _PyAccess32_4, - 'F': _PyAccessF, - 'I': _PyAccessI32_N, - } +mode_map = { + "1": _PyAccess8, + "L": _PyAccess8, + "P": _PyAccess8, + "LA": _PyAccess32_2, + "La": _PyAccess32_2, + "PA": _PyAccess32_2, + "RGB": _PyAccess32_3, + "LAB": _PyAccess32_3, + "HSV": _PyAccess32_3, + "YCbCr": _PyAccess32_3, + "RGBA": _PyAccess32_4, + "RGBa": _PyAccess32_4, + "RGBX": _PyAccess32_4, + "CMYK": _PyAccess32_4, + "F": _PyAccessF, + "I": _PyAccessI32_N, +} -if sys.byteorder == 'little': - mode_map['I;16'] = _PyAccessI16_N - mode_map['I;16L'] = _PyAccessI16_N - mode_map['I;16B'] = _PyAccessI16_B +if sys.byteorder == "little": + mode_map["I;16"] = _PyAccessI16_N + mode_map["I;16L"] = _PyAccessI16_N + mode_map["I;16B"] = _PyAccessI16_B - mode_map['I;32L'] = _PyAccessI32_N - mode_map['I;32B'] = _PyAccessI32_Swap + mode_map["I;32L"] = _PyAccessI32_N + mode_map["I;32B"] = _PyAccessI32_Swap else: - mode_map['I;16'] = _PyAccessI16_L - mode_map['I;16L'] = _PyAccessI16_L - mode_map['I;16B'] = _PyAccessI16_N + mode_map["I;16"] = _PyAccessI16_L + mode_map["I;16L"] = _PyAccessI16_L + mode_map["I;16B"] = _PyAccessI16_N - mode_map['I;32L'] = _PyAccessI32_Swap - mode_map['I;32B'] = _PyAccessI32_N + mode_map["I;32L"] = _PyAccessI32_Swap + mode_map["I;32B"] = _PyAccessI32_N def new(img, readonly=False): diff --git a/server/www/packages/packages-linux/x64/PIL/SgiImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/SgiImagePlugin.py index 88df351..99408fd 100644 --- a/server/www/packages/packages-linux/x64/PIL/SgiImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/SgiImagePlugin.py @@ -22,13 +22,15 @@ # -from . import Image, ImageFile -from ._binary import i8, o8, i16be as i16 -from ._util import py3 -import struct import os +import struct +from . import Image, ImageFile +from ._binary import i8, i16be as i16, o8 +from ._util import py3 +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.3" @@ -44,7 +46,7 @@ MODES = { (1, 3, 3): "RGB", (2, 3, 3): "RGB;16B", (1, 3, 4): "RGBA", - (2, 3, 4): "RGBA;16B" + (2, 3, 4): "RGBA;16B", } @@ -98,6 +100,8 @@ class SgiImageFile(ImageFile.ImageFile): self._size = xsize, ysize self.mode = rawmode.split(";")[0] + if self.mode == "RGB": + self.custom_mimetype = "image/rgb" # orientation -1 : scanlines begins at the bottom-left corner orientation = -1 @@ -106,19 +110,21 @@ class SgiImageFile(ImageFile.ImageFile): if compression == 0: pagesize = xsize * ysize * bpc if bpc == 2: - self.tile = [("SGI16", (0, 0) + self.size, - headlen, (self.mode, 0, orientation))] + self.tile = [ + ("SGI16", (0, 0) + self.size, headlen, (self.mode, 0, orientation)) + ] else: self.tile = [] offset = headlen for layer in self.mode: self.tile.append( - ("raw", (0, 0) + self.size, - offset, (layer, 0, orientation))) + ("raw", (0, 0) + self.size, offset, (layer, 0, orientation)) + ) offset += pagesize elif compression == 1: - self.tile = [("sgi_rle", (0, 0) + self.size, - headlen, (rawmode, orientation, bpc))] + self.tile = [ + ("sgi_rle", (0, 0) + self.size, headlen, (rawmode, orientation, bpc)) + ] def _save(im, fp, filename): @@ -157,8 +163,9 @@ def _save(im, fp, filename): # assert we've got the right number of bands. if len(im.getbands()) != z: - raise ValueError("incorrect number of bands in SGI write: %s vs %s" % - (z, len(im.getbands()))) + raise ValueError( + "incorrect number of bands in SGI write: %s vs %s" % (z, len(im.getbands())) + ) # Minimum Byte value pinmin = 0 @@ -167,30 +174,30 @@ def _save(im, fp, filename): # Image name (79 characters max, truncated below in write) imgName = os.path.splitext(os.path.basename(filename))[0] if py3: - imgName = imgName.encode('ascii', 'ignore') + imgName = imgName.encode("ascii", "ignore") # Standard representation of pixel in the file colormap = 0 - fp.write(struct.pack('>h', magicNumber)) + fp.write(struct.pack(">h", magicNumber)) fp.write(o8(rle)) fp.write(o8(bpc)) - fp.write(struct.pack('>H', dim)) - fp.write(struct.pack('>H', x)) - fp.write(struct.pack('>H', y)) - fp.write(struct.pack('>H', z)) - fp.write(struct.pack('>l', pinmin)) - fp.write(struct.pack('>l', pinmax)) - fp.write(struct.pack('4s', b'')) # dummy - fp.write(struct.pack('79s', imgName)) # truncates to 79 chars - fp.write(struct.pack('s', b'')) # force null byte after imgname - fp.write(struct.pack('>l', colormap)) - fp.write(struct.pack('404s', b'')) # dummy + fp.write(struct.pack(">H", dim)) + fp.write(struct.pack(">H", x)) + fp.write(struct.pack(">H", y)) + fp.write(struct.pack(">H", z)) + fp.write(struct.pack(">l", pinmin)) + fp.write(struct.pack(">l", pinmax)) + fp.write(struct.pack("4s", b"")) # dummy + fp.write(struct.pack("79s", imgName)) # truncates to 79 chars + fp.write(struct.pack("s", b"")) # force null byte after imgname + fp.write(struct.pack(">l", colormap)) + fp.write(struct.pack("404s", b"")) # dummy - rawmode = 'L' + rawmode = "L" if bpc == 2: - rawmode = 'L;16B' + rawmode = "L;16B" for channel in im.split(): - fp.write(channel.tobytes('raw', rawmode, 0, orientation)) + fp.write(channel.tobytes("raw", rawmode, 0, orientation)) fp.close() @@ -205,13 +212,15 @@ class SGI16Decoder(ImageFile.PyDecoder): self.fd.seek(512) for band in range(zsize): - channel = Image.new('L', (self.state.xsize, self.state.ysize)) - channel.frombytes(self.fd.read(2 * pagesize), 'raw', - 'L;16B', stride, orientation) + channel = Image.new("L", (self.state.xsize, self.state.ysize)) + channel.frombytes( + self.fd.read(2 * pagesize), "raw", "L;16B", stride, orientation + ) self.im.putband(channel.im, band) return -1, 0 + # # registry @@ -220,9 +229,7 @@ Image.register_decoder("SGI16", SGI16Decoder) Image.register_open(SgiImageFile.format, SgiImageFile, _accept) Image.register_save(SgiImageFile.format, _save) Image.register_mime(SgiImageFile.format, "image/sgi") -Image.register_mime(SgiImageFile.format, "image/rgb") -Image.register_extensions(SgiImageFile.format, - [".bw", ".rgb", ".rgba", ".sgi"]) +Image.register_extensions(SgiImageFile.format, [".bw", ".rgb", ".rgba", ".sgi"]) # End of file diff --git a/server/www/packages/packages-linux/x64/PIL/SpiderImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/SpiderImagePlugin.py index 3f57952..f1cae4d 100644 --- a/server/www/packages/packages-linux/x64/PIL/SpiderImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/SpiderImagePlugin.py @@ -35,16 +35,17 @@ from __future__ import print_function -from PIL import Image, ImageFile import os import struct import sys +from PIL import Image, ImageFile + def isInt(f): try: i = int(f) - if f-i == 0: + if f - i == 0: return 1 else: return 0 @@ -60,8 +61,9 @@ iforms = [1, 3, -11, -12, -21, -22] # Returns no. of bytes in the header, if it is a valid Spider header, # otherwise returns 0 + def isSpiderHeader(t): - h = (99,) + t # add 1 value so can use spider header index start=1 + h = (99,) + t # add 1 value so can use spider header index start=1 # header values 1,2,5,12,13,22,23 should be integers for i in [1, 2, 5, 12, 13, 22, 23]: if not isInt(h[i]): @@ -71,9 +73,9 @@ def isSpiderHeader(t): if iform not in iforms: return 0 # check other header values - labrec = int(h[13]) # no. records in file header - labbyt = int(h[22]) # total no. of bytes in header - lenbyt = int(h[23]) # record length in bytes + labrec = int(h[13]) # no. records in file header + labbyt = int(h[22]) # total no. of bytes in header + lenbyt = int(h[23]) # record length in bytes if labbyt != (labrec * lenbyt): return 0 # looks like a valid header @@ -81,12 +83,12 @@ def isSpiderHeader(t): def isSpiderImage(filename): - with open(filename, 'rb') as fp: - f = fp.read(92) # read 23 * 4 bytes - t = struct.unpack('>23f', f) # try big-endian first + with open(filename, "rb") as fp: + f = fp.read(92) # read 23 * 4 bytes + t = struct.unpack(">23f", f) # try big-endian first hdrlen = isSpiderHeader(t) if hdrlen == 0: - t = struct.unpack('<23f', f) # little-endian + t = struct.unpack("<23f", f) # little-endian hdrlen = isSpiderHeader(t) return hdrlen @@ -104,18 +106,18 @@ class SpiderImageFile(ImageFile.ImageFile): try: self.bigendian = 1 - t = struct.unpack('>27f', f) # try big-endian first + t = struct.unpack(">27f", f) # try big-endian first hdrlen = isSpiderHeader(t) if hdrlen == 0: self.bigendian = 0 - t = struct.unpack('<27f', f) # little-endian + t = struct.unpack("<27f", f) # little-endian hdrlen = isSpiderHeader(t) if hdrlen == 0: raise SyntaxError("not a valid Spider file") except struct.error: raise SyntaxError("not a valid Spider file") - h = (99,) + t # add 1 value : spider header index starts at 1 + h = (99,) + t # add 1 value : spider header index starts at 1 iform = int(h[5]) if iform != 1: raise SyntaxError("not a Spider 2D image") @@ -149,9 +151,7 @@ class SpiderImageFile(ImageFile.ImageFile): self.rawmode = "F;32F" self.mode = "F" - self.tile = [ - ("raw", (0, 0) + self.size, offset, - (self.rawmode, 0, 1))] + self.tile = [("raw", (0, 0) + self.size, offset, (self.rawmode, 0, 1))] self.__fp = self.fp # FIXME: hack @property @@ -184,22 +184,32 @@ class SpiderImageFile(ImageFile.ImageFile): (minimum, maximum) = self.getextrema() m = 1 if maximum != minimum: - m = depth / (maximum-minimum) + m = depth / (maximum - minimum) b = -m * minimum return self.point(lambda i, m=m, b=b: i * m + b).convert("L") # returns a ImageTk.PhotoImage object, after rescaling to 0..255 def tkPhotoImage(self): from PIL import ImageTk + return ImageTk.PhotoImage(self.convert2byte(), palette=256) + def _close__fp(self): + try: + if self.__fp != self.fp: + self.__fp.close() + except AttributeError: + pass + finally: + self.__fp = None + # -------------------------------------------------------------------- # Image series # given a list of filenames, return a list of images def loadImageSeries(filelist=None): - """create a list of Image.images for use in montage""" + """create a list of :py:class:`~PIL.Image.Image` objects for use in a montage""" if filelist is None or len(filelist) < 1: return @@ -210,11 +220,11 @@ def loadImageSeries(filelist=None): continue try: im = Image.open(img).convert2byte() - except: + except Exception: if not isSpiderImage(img): print(img + " is not a Spider image file") continue - im.info['filename'] = img + im.info["filename"] = img imglist.append(im) return imglist @@ -222,10 +232,11 @@ def loadImageSeries(filelist=None): # -------------------------------------------------------------------- # For saving images in Spider format + def makeSpiderHeader(im): nsam, nrow = im.size lenbyt = nsam * 4 # There are labrec records in the header - labrec = 1024 / lenbyt + labrec = int(1024 / lenbyt) if 1024 % lenbyt != 0: labrec += 1 labbyt = labrec * lenbyt @@ -238,10 +249,10 @@ def makeSpiderHeader(im): return [] # NB these are Fortran indices - hdr[1] = 1.0 # nslice (=1 for an image) - hdr[2] = float(nrow) # number of rows per slice - hdr[5] = 1.0 # iform for 2D image - hdr[12] = float(nsam) # number of pixels per line + hdr[1] = 1.0 # nslice (=1 for an image) + hdr[2] = float(nrow) # number of rows per slice + hdr[5] = 1.0 # iform for 2D image + hdr[12] = float(nsam) # number of pixels per line hdr[13] = float(labrec) # number of records in file header hdr[22] = float(labbyt) # total number of bytes in header hdr[23] = float(lenbyt) # record length in bytes @@ -252,13 +263,13 @@ def makeSpiderHeader(im): # pack binary data into a string hdrstr = [] for v in hdr: - hdrstr.append(struct.pack('f', v)) + hdrstr.append(struct.pack("f", v)) return hdrstr def _save(im, fp, filename): if im.mode[0] != "F": - im = im.convert('F') + im = im.convert("F") hdr = makeSpiderHeader(im) if len(hdr) < 256: @@ -268,7 +279,7 @@ def _save(im, fp, filename): fp.writelines(hdr) rawmode = "F;32NF" # 32-bit native floating point - ImageFile._save(im, fp, [("raw", (0, 0)+im.size, 0, (rawmode, 0, 1))]) + ImageFile._save(im, fp, [("raw", (0, 0) + im.size, 0, (rawmode, 0, 1))]) def _save_spider(im, fp, filename): @@ -277,6 +288,7 @@ def _save_spider(im, fp, filename): Image.register_extension(SpiderImageFile.format, ext) _save(im, fp, filename) + # -------------------------------------------------------------------- @@ -299,7 +311,7 @@ if __name__ == "__main__": print("format: " + str(im.format)) print("size: " + str(im.size)) print("mode: " + str(im.mode)) - print("max, min: ", end=' ') + print("max, min: ", end=" ") print(im.getextrema()) if len(sys.argv) > 2: @@ -308,6 +320,7 @@ if __name__ == "__main__": # perform some image operation im = im.transpose(Image.FLIP_LEFT_RIGHT) print( - "saving a flipped version of %s as %s " % - (os.path.basename(filename), outfile)) + "saving a flipped version of %s as %s " + % (os.path.basename(filename), outfile) + ) im.save(outfile, SpiderImageFile.format) diff --git a/server/www/packages/packages-linux/x64/PIL/SunImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/SunImagePlugin.py index 898350e..74fa5f7 100644 --- a/server/www/packages/packages-linux/x64/PIL/SunImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/SunImagePlugin.py @@ -20,16 +20,19 @@ from . import Image, ImageFile, ImagePalette from ._binary import i32be as i32 +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.3" def _accept(prefix): - return len(prefix) >= 4 and i32(prefix) == 0x59a66a95 + return len(prefix) >= 4 and i32(prefix) == 0x59A66A95 ## # Image plugin for Sun raster files. + class SunImageFile(ImageFile.ImageFile): format = "SUN" @@ -54,7 +57,7 @@ class SunImageFile(ImageFile.ImageFile): # HEAD s = self.fp.read(32) - if i32(s) != 0x59a66a95: + if i32(s) != 0x59A66A95: raise SyntaxError("not an SUN raster file") offset = 32 @@ -80,9 +83,9 @@ class SunImageFile(ImageFile.ImageFile): self.mode, rawmode = "RGB", "BGR" elif depth == 32: if file_type == 3: - self.mode, rawmode = 'RGB', 'RGBX' + self.mode, rawmode = "RGB", "RGBX" else: - self.mode, rawmode = 'RGB', 'BGRX' + self.mode, rawmode = "RGB", "BGRX" else: raise SyntaxError("Unsupported Mode/Bit Depth") @@ -94,11 +97,10 @@ class SunImageFile(ImageFile.ImageFile): raise SyntaxError("Unsupported Palette Type") offset = offset + palette_length - self.palette = ImagePalette.raw("RGB;L", - self.fp.read(palette_length)) + self.palette = ImagePalette.raw("RGB;L", self.fp.read(palette_length)) if self.mode == "L": self.mode = "P" - rawmode = rawmode.replace('L', 'P') + rawmode = rawmode.replace("L", "P") # 16 bit boundaries on stride stride = ((self.size[0] * depth + 15) // 16) * 2 @@ -122,11 +124,12 @@ class SunImageFile(ImageFile.ImageFile): # (https://www.fileformat.info/format/sunraster/egff.htm) if file_type in (0, 1, 3, 4, 5): - self.tile = [("raw", (0, 0)+self.size, offset, (rawmode, stride))] + self.tile = [("raw", (0, 0) + self.size, offset, (rawmode, stride))] elif file_type == 2: - self.tile = [("sun_rle", (0, 0)+self.size, offset, rawmode)] + self.tile = [("sun_rle", (0, 0) + self.size, offset, rawmode)] else: - raise SyntaxError('Unsupported Sun Raster file type') + raise SyntaxError("Unsupported Sun Raster file type") + # # registry diff --git a/server/www/packages/packages-linux/x64/PIL/TarIO.py b/server/www/packages/packages-linux/x64/PIL/TarIO.py index 0e949ff..e180b80 100644 --- a/server/www/packages/packages-linux/x64/PIL/TarIO.py +++ b/server/www/packages/packages-linux/x64/PIL/TarIO.py @@ -14,15 +14,17 @@ # See the README file for information on usage and redistribution. # -from . import ContainerIO +import io +import sys +from . import ContainerIO ## # A file object that provides read access to a given member of a TAR # file. -class TarIO(ContainerIO.ContainerIO): +class TarIO(ContainerIO.ContainerIO): def __init__(self, tarfile, file): """ Create file object. @@ -30,16 +32,16 @@ class TarIO(ContainerIO.ContainerIO): :param tarfile: Name of TAR file. :param file: Name of member file. """ - fh = open(tarfile, "rb") + self.fh = open(tarfile, "rb") while True: - s = fh.read(512) + s = self.fh.read(512) if len(s) != 512: raise IOError("unexpected end of tar file") - name = s[:100].decode('utf-8') - i = name.find('\0') + name = s[:100].decode("utf-8") + i = name.find("\0") if i == 0: raise IOError("cannot find subfile") if i > 0: @@ -50,7 +52,22 @@ class TarIO(ContainerIO.ContainerIO): if file == name: break - fh.seek((size + 511) & (~511), 1) + self.fh.seek((size + 511) & (~511), io.SEEK_CUR) # Open region - ContainerIO.ContainerIO.__init__(self, fh, fh.tell(), size) + ContainerIO.ContainerIO.__init__(self, self.fh, self.fh.tell(), size) + + # Context manager support + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + if sys.version_info.major >= 3: + + def __del__(self): + self.close() + + def close(self): + self.fh.close() diff --git a/server/www/packages/packages-linux/x64/PIL/TgaImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/TgaImagePlugin.py index c266d50..b1b3513 100644 --- a/server/www/packages/packages-linux/x64/PIL/TgaImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/TgaImagePlugin.py @@ -17,11 +17,13 @@ # +import warnings + from . import Image, ImageFile, ImagePalette from ._binary import i8, i16le as i16, o8, o16le as o16 -import warnings - +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.3" @@ -32,9 +34,9 @@ __version__ = "0.3" MODES = { # map imagetype/depth to rawmode - (1, 8): "P", - (3, 1): "1", - (3, 8): "L", + (1, 8): "P", + (3, 1): "1", + (3, 8): "L", (3, 16): "LA", (2, 16): "BGR;5", (2, 24): "BGR", @@ -45,6 +47,7 @@ MODES = { ## # Image plugin for Targa files. + class TgaImageFile(ImageFile.ImageFile): format = "TGA" @@ -67,9 +70,12 @@ class TgaImageFile(ImageFile.ImageFile): self._size = i16(s[12:]), i16(s[14:]) # validate header fields - if colormaptype not in (0, 1) or\ - self.size[0] <= 0 or self.size[1] <= 0 or\ - depth not in (1, 8, 16, 24, 32): + if ( + colormaptype not in (0, 1) + or self.size[0] <= 0 + or self.size[1] <= 0 + or depth not in (1, 8, 16, 24, 32) + ): raise SyntaxError("not a TGA file") # image mode @@ -110,27 +116,43 @@ class TgaImageFile(ImageFile.ImageFile): start, size, mapdepth = i16(s[3:]), i16(s[5:]), i16(s[7:]) if mapdepth == 16: self.palette = ImagePalette.raw( - "BGR;16", b"\0"*2*start + self.fp.read(2*size)) + "BGR;16", b"\0" * 2 * start + self.fp.read(2 * size) + ) elif mapdepth == 24: self.palette = ImagePalette.raw( - "BGR", b"\0"*3*start + self.fp.read(3*size)) + "BGR", b"\0" * 3 * start + self.fp.read(3 * size) + ) elif mapdepth == 32: self.palette = ImagePalette.raw( - "BGRA", b"\0"*4*start + self.fp.read(4*size)) + "BGRA", b"\0" * 4 * start + self.fp.read(4 * size) + ) # setup tile descriptor try: rawmode = MODES[(imagetype & 7, depth)] if imagetype & 8: # compressed - self.tile = [("tga_rle", (0, 0)+self.size, - self.fp.tell(), (rawmode, orientation, depth))] + self.tile = [ + ( + "tga_rle", + (0, 0) + self.size, + self.fp.tell(), + (rawmode, orientation, depth), + ) + ] else: - self.tile = [("raw", (0, 0)+self.size, - self.fp.tell(), (rawmode, 0, orientation))] + self.tile = [ + ( + "raw", + (0, 0) + self.size, + self.fp.tell(), + (rawmode, 0, orientation), + ) + ] except KeyError: pass # cannot decode + # # -------------------------------------------------------------------- # Write TGA file @@ -156,14 +178,12 @@ def _save(im, fp, filename): if "rle" in im.encoderinfo: rle = im.encoderinfo["rle"] else: - compression = im.encoderinfo.get("compression", - im.info.get("compression")) + compression = im.encoderinfo.get("compression", im.info.get("compression")) rle = compression == "tga_rle" if rle: imagetype += 8 - id_section = im.encoderinfo.get("id_section", - im.info.get("id_section", "")) + id_section = im.encoderinfo.get("id_section", im.info.get("id_section", "")) id_len = len(id_section) if id_len > 255: id_len = 255 @@ -180,23 +200,24 @@ def _save(im, fp, filename): else: flags = 0 - orientation = im.encoderinfo.get("orientation", - im.info.get("orientation", -1)) + orientation = im.encoderinfo.get("orientation", im.info.get("orientation", -1)) if orientation > 0: flags = flags | 0x20 - fp.write(o8(id_len) + - o8(colormaptype) + - o8(imagetype) + - o16(colormapfirst) + - o16(colormaplength) + - o8(colormapentry) + - o16(0) + - o16(0) + - o16(im.size[0]) + - o16(im.size[1]) + - o8(bits) + - o8(flags)) + fp.write( + o8(id_len) + + o8(colormaptype) + + o8(imagetype) + + o16(colormapfirst) + + o16(colormaplength) + + o8(colormapentry) + + o16(0) + + o16(0) + + o16(im.size[0]) + + o16(im.size[1]) + + o8(bits) + + o8(flags) + ) if id_section: fp.write(id_section) @@ -206,16 +227,17 @@ def _save(im, fp, filename): if rle: ImageFile._save( - im, - fp, - [("tga_rle", (0, 0) + im.size, 0, (rawmode, orientation))]) + im, fp, [("tga_rle", (0, 0) + im.size, 0, (rawmode, orientation))] + ) else: ImageFile._save( - im, fp, [("raw", (0, 0) + im.size, 0, (rawmode, 0, orientation))]) + im, fp, [("raw", (0, 0) + im.size, 0, (rawmode, 0, orientation))] + ) # write targa version 2 footer fp.write(b"\000" * 8 + b"TRUEVISION-XFILE." + b"\000") + # # -------------------------------------------------------------------- # Registry @@ -224,4 +246,6 @@ def _save(im, fp, filename): Image.register_open(TgaImageFile.format, TgaImageFile) Image.register_save(TgaImageFile.format, _save) -Image.register_extension(TgaImageFile.format, ".tga") +Image.register_extensions(TgaImageFile.format, [".tga", ".icb", ".vda", ".vst"]) + +Image.register_mime(TgaImageFile.format, "image/x-tga") diff --git a/server/www/packages/packages-linux/x64/PIL/TiffImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/TiffImagePlugin.py index 5059a13..a927cd3 100644 --- a/server/www/packages/packages-linux/x64/PIL/TiffImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/TiffImagePlugin.py @@ -41,20 +41,19 @@ from __future__ import division, print_function -from . import Image, ImageFile, ImagePalette, TiffTags -from ._binary import i8, o8 -from ._util import py3 - -from fractions import Fraction -from numbers import Number, Rational - +import distutils.version import io import itertools import os import struct import sys import warnings +from fractions import Fraction +from numbers import Number, Rational +from . import Image, ImageFile, ImagePalette, TiffTags +from ._binary import i8, o8 +from ._util import py3 from .TiffTags import TYPES try: @@ -65,6 +64,8 @@ except ImportError: from collections import MutableMapping +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "1.3.5" DEBUG = False # Needs to be merged with the new logging approach. @@ -96,6 +97,7 @@ X_RESOLUTION = 282 Y_RESOLUTION = 283 PLANAR_CONFIGURATION = 284 RESOLUTION_UNIT = 296 +TRANSFERFUNCTION = 301 SOFTWARE = 305 DATE_TIME = 306 ARTIST = 315 @@ -105,12 +107,14 @@ TILEOFFSETS = 324 EXTRASAMPLES = 338 SAMPLEFORMAT = 339 JPEGTABLES = 347 +REFERENCEBLACKWHITE = 532 COPYRIGHT = 33432 IPTC_NAA_CHUNK = 33723 # newsphoto properties PHOTOSHOP_CHUNK = 34377 # photoshop properties ICCPROFILE = 34675 EXIFIFD = 34665 XMP = 700 +JPEGQUALITY = 65537 # pseudo-tag by libtiff # https://github.com/imagej/ImageJA/blob/master/src/main/java/ij/io/TiffDecoder.java IMAGEJ_META_DATA_BYTE_COUNTS = 50838 @@ -132,6 +136,9 @@ COMPRESSION_INFO = { 32946: "tiff_deflate", 34676: "tiff_sgilog", 34677: "tiff_sgilog24", + 34925: "lzma", + 50000: "zstd", + 50001: "webp", } COMPRESSION_INFO_REV = {v: k for k, v in COMPRESSION_INFO.items()} @@ -147,7 +154,6 @@ OPEN_INFO = { (MM, 1, (1,), 1, (1,), ()): ("1", "1"), (II, 1, (1,), 2, (1,), ()): ("1", "1;R"), (MM, 1, (1,), 2, (1,), ()): ("1", "1;R"), - (II, 0, (1,), 1, (2,), ()): ("L", "L;2I"), (MM, 0, (1,), 1, (2,), ()): ("L", "L;2I"), (II, 0, (1,), 2, (2,), ()): ("L", "L;2IR"), @@ -156,7 +162,6 @@ OPEN_INFO = { (MM, 1, (1,), 1, (2,), ()): ("L", "L;2"), (II, 1, (1,), 2, (2,), ()): ("L", "L;2R"), (MM, 1, (1,), 2, (2,), ()): ("L", "L;2R"), - (II, 0, (1,), 1, (4,), ()): ("L", "L;4I"), (MM, 0, (1,), 1, (4,), ()): ("L", "L;4I"), (II, 0, (1,), 2, (4,), ()): ("L", "L;4IR"), @@ -165,7 +170,6 @@ OPEN_INFO = { (MM, 1, (1,), 1, (4,), ()): ("L", "L;4"), (II, 1, (1,), 2, (4,), ()): ("L", "L;4R"), (MM, 1, (1,), 2, (4,), ()): ("L", "L;4R"), - (II, 0, (1,), 1, (8,), ()): ("L", "L;I"), (MM, 0, (1,), 1, (8,), ()): ("L", "L;I"), (II, 0, (1,), 2, (8,), ()): ("L", "L;IR"), @@ -174,14 +178,11 @@ OPEN_INFO = { (MM, 1, (1,), 1, (8,), ()): ("L", "L"), (II, 1, (1,), 2, (8,), ()): ("L", "L;R"), (MM, 1, (1,), 2, (8,), ()): ("L", "L;R"), - (II, 1, (1,), 1, (12,), ()): ("I;16", "I;12"), - (II, 1, (1,), 1, (16,), ()): ("I;16", "I;16"), (MM, 1, (1,), 1, (16,), ()): ("I;16B", "I;16B"), (II, 1, (2,), 1, (16,), ()): ("I", "I;16S"), (MM, 1, (2,), 1, (16,), ()): ("I", "I;16BS"), - (II, 0, (3,), 1, (32,), ()): ("F", "F;32F"), (MM, 0, (3,), 1, (32,), ()): ("F", "F;32BF"), (II, 1, (1,), 1, (32,), ()): ("I", "I;32N"), @@ -189,10 +190,8 @@ OPEN_INFO = { (MM, 1, (2,), 1, (32,), ()): ("I", "I;32BS"), (II, 1, (3,), 1, (32,), ()): ("F", "F;32F"), (MM, 1, (3,), 1, (32,), ()): ("F", "F;32BF"), - (II, 1, (1,), 1, (8, 8), (2,)): ("LA", "LA"), (MM, 1, (1,), 1, (8, 8), (2,)): ("LA", "LA"), - (II, 2, (1,), 1, (8, 8, 8), ()): ("RGB", "RGB"), (MM, 2, (1,), 1, (8, 8, 8), ()): ("RGB", "RGB"), (II, 2, (1,), 2, (8, 8, 8), ()): ("RGB", "RGB;R"), @@ -219,7 +218,6 @@ OPEN_INFO = { (MM, 2, (1,), 1, (8, 8, 8, 8, 8, 8), (2, 0, 0)): ("RGBA", "RGBAXX"), (II, 2, (1,), 1, (8, 8, 8, 8), (999,)): ("RGBA", "RGBA"), # Corel Draw 10 (MM, 2, (1,), 1, (8, 8, 8, 8), (999,)): ("RGBA", "RGBA"), # Corel Draw 10 - (II, 2, (1,), 1, (16, 16, 16), ()): ("RGB", "RGB;16L"), (MM, 2, (1,), 1, (16, 16, 16), ()): ("RGB", "RGB;16B"), (II, 2, (1,), 1, (16, 16, 16, 16), ()): ("RGBA", "RGBA;16L"), @@ -230,7 +228,6 @@ OPEN_INFO = { (MM, 2, (1,), 1, (16, 16, 16, 16), (1,)): ("RGBA", "RGBa;16B"), (II, 2, (1,), 1, (16, 16, 16, 16), (2,)): ("RGBA", "RGBA;16L"), (MM, 2, (1,), 1, (16, 16, 16, 16), (2,)): ("RGBA", "RGBA;16B"), - (II, 3, (1,), 1, (1,), ()): ("P", "P;1"), (MM, 3, (1,), 1, (1,), ()): ("P", "P;1"), (II, 3, (1,), 2, (1,), ()): ("P", "P;1R"), @@ -249,19 +246,17 @@ OPEN_INFO = { (MM, 3, (1,), 1, (8, 8), (2,)): ("PA", "PA"), (II, 3, (1,), 2, (8,), ()): ("P", "P;R"), (MM, 3, (1,), 2, (8,), ()): ("P", "P;R"), - (II, 5, (1,), 1, (8, 8, 8, 8), ()): ("CMYK", "CMYK"), (MM, 5, (1,), 1, (8, 8, 8, 8), ()): ("CMYK", "CMYK"), (II, 5, (1,), 1, (8, 8, 8, 8, 8), (0,)): ("CMYK", "CMYKX"), (MM, 5, (1,), 1, (8, 8, 8, 8, 8), (0,)): ("CMYK", "CMYKX"), (II, 5, (1,), 1, (8, 8, 8, 8, 8, 8), (0, 0)): ("CMYK", "CMYKXX"), (MM, 5, (1,), 1, (8, 8, 8, 8, 8, 8), (0, 0)): ("CMYK", "CMYKXX"), - - # JPEG compressed images handled by LibTiff and auto-converted to RGB + (II, 5, (1,), 1, (16, 16, 16, 16), ()): ("CMYK", "CMYK;16L"), + # JPEG compressed images handled by LibTiff and auto-converted to RGBX # Minimal Baseline TIFF requires YCbCr images to have 3 SamplesPerPixel - (II, 6, (1,), 1, (8, 8, 8), ()): ("RGB", "RGB"), - (MM, 6, (1,), 1, (8, 8, 8), ()): ("RGB", "RGB"), - + (II, 6, (1,), 1, (8, 8, 8), ()): ("RGB", "RGBX"), + (MM, 6, (1,), 1, (8, 8, 8), ()): ("RGB", "RGBX"), (II, 8, (1,), 1, (8, 8, 8), ()): ("LAB", "LAB"), (MM, 8, (1,), 1, (8, 8, 8), ()): ("LAB", "LAB"), } @@ -284,6 +279,10 @@ def _limit_rational(val, max_val): return n_d[::-1] if inv else n_d +def _libtiff_version(): + return Image.core.libtiff_version.split("\n")[0].split("Version ")[1] + + ## # Wrapper for TIFF IFDs. @@ -303,7 +302,7 @@ class IFDRational(Rational): """ - __slots__ = ('_numerator', '_denominator', '_val') + __slots__ = ("_numerator", "_denominator", "_val") def __init__(self, value, denominator=1): """ @@ -327,7 +326,7 @@ class IFDRational(Rational): return if denominator == 0: - self._val = float('nan') + self._val = float("nan") return elif denominator == 1: @@ -368,6 +367,7 @@ class IFDRational(Rational): def _delegate(op): def delegate(self, *args): return getattr(self._val, op)(*args) + return delegate """ a = ['add','radd', 'sub', 'rsub','div', 'rdiv', 'mul', 'rmul', @@ -378,34 +378,34 @@ class IFDRational(Rational): print("\n".join("__%s__ = _delegate('__%s__')" % (s,s) for s in a)) """ - __add__ = _delegate('__add__') - __radd__ = _delegate('__radd__') - __sub__ = _delegate('__sub__') - __rsub__ = _delegate('__rsub__') - __div__ = _delegate('__div__') - __rdiv__ = _delegate('__rdiv__') - __mul__ = _delegate('__mul__') - __rmul__ = _delegate('__rmul__') - __truediv__ = _delegate('__truediv__') - __rtruediv__ = _delegate('__rtruediv__') - __floordiv__ = _delegate('__floordiv__') - __rfloordiv__ = _delegate('__rfloordiv__') - __mod__ = _delegate('__mod__') - __rmod__ = _delegate('__rmod__') - __pow__ = _delegate('__pow__') - __rpow__ = _delegate('__rpow__') - __pos__ = _delegate('__pos__') - __neg__ = _delegate('__neg__') - __abs__ = _delegate('__abs__') - __trunc__ = _delegate('__trunc__') - __lt__ = _delegate('__lt__') - __gt__ = _delegate('__gt__') - __le__ = _delegate('__le__') - __ge__ = _delegate('__ge__') - __nonzero__ = _delegate('__nonzero__') - __ceil__ = _delegate('__ceil__') - __floor__ = _delegate('__floor__') - __round__ = _delegate('__round__') + __add__ = _delegate("__add__") + __radd__ = _delegate("__radd__") + __sub__ = _delegate("__sub__") + __rsub__ = _delegate("__rsub__") + __div__ = _delegate("__div__") + __rdiv__ = _delegate("__rdiv__") + __mul__ = _delegate("__mul__") + __rmul__ = _delegate("__rmul__") + __truediv__ = _delegate("__truediv__") + __rtruediv__ = _delegate("__rtruediv__") + __floordiv__ = _delegate("__floordiv__") + __rfloordiv__ = _delegate("__rfloordiv__") + __mod__ = _delegate("__mod__") + __rmod__ = _delegate("__rmod__") + __pow__ = _delegate("__pow__") + __rpow__ = _delegate("__rpow__") + __pos__ = _delegate("__pos__") + __neg__ = _delegate("__neg__") + __abs__ = _delegate("__abs__") + __trunc__ = _delegate("__trunc__") + __lt__ = _delegate("__lt__") + __gt__ = _delegate("__gt__") + __le__ = _delegate("__le__") + __ge__ = _delegate("__ge__") + __nonzero__ = _delegate("__nonzero__") + __ceil__ = _delegate("__ceil__") + __floor__ = _delegate("__floor__") + __round__ = _delegate("__round__") class ImageFileDirectory_v2(MutableMapping): @@ -416,7 +416,7 @@ class ImageFileDirectory_v2(MutableMapping): ifd = ImageFileDirectory_v2() ifd[key] = 'Some Data' - ifd.tagtype[key] = 2 + ifd.tagtype[key] = TiffTags.ASCII print(ifd[key]) 'Some Data' @@ -439,6 +439,7 @@ class ImageFileDirectory_v2(MutableMapping): .. versionadded:: 3.0.0 """ + """ Documentation: @@ -498,7 +499,7 @@ class ImageFileDirectory_v2(MutableMapping): self._tags_v1 = {} # will remain empty if legacy_api is false self._tags_v2 = {} # main tag storage self._tagdata = {} - self.tagtype = {} # added 2008-06-05 by Florian Hoech + self.tagtype = {} # added 2008-06-05 by Florian Hoech self._next = None self._offset = None @@ -511,8 +512,7 @@ class ImageFileDirectory_v2(MutableMapping): Returns the complete tag dictionary, with named tags where possible. """ - return dict((TiffTags.lookup(code).name, value) - for code, value in self.items()) + return {TiffTags.lookup(code).name: value for code, value in self.items()} def __len__(self): return len(set(self._tagdata) | set(self._tags_v2)) @@ -525,13 +525,14 @@ class ImageFileDirectory_v2(MutableMapping): self[tag] = handler(self, data, self.legacy_api) # check type val = self._tags_v2[tag] if self.legacy_api and not isinstance(val, (tuple, bytes)): - val = val, + val = (val,) return val def __contains__(self, tag): return tag in self._tags_v2 or tag in self._tagdata if not py3: + def has_key(self, tag): return tag in self @@ -541,7 +542,7 @@ class ImageFileDirectory_v2(MutableMapping): def _setitem(self, tag, value, legacy_api): basetypes = (Number, bytes, str) if not py3: - basetypes += unicode, + basetypes += (unicode,) # noqa: F821 info = TiffTags.lookup(tag) values = [value] if isinstance(value, basetypes) else value @@ -550,30 +551,30 @@ class ImageFileDirectory_v2(MutableMapping): if info.type: self.tagtype[tag] = info.type else: - self.tagtype[tag] = 7 + self.tagtype[tag] = TiffTags.UNDEFINED if all(isinstance(v, IFDRational) for v in values): - self.tagtype[tag] = 5 + self.tagtype[tag] = TiffTags.RATIONAL elif all(isinstance(v, int) for v in values): if all(v < 2 ** 16 for v in values): - self.tagtype[tag] = 3 + self.tagtype[tag] = TiffTags.SHORT else: - self.tagtype[tag] = 4 + self.tagtype[tag] = TiffTags.LONG elif all(isinstance(v, float) for v in values): - self.tagtype[tag] = 12 + self.tagtype[tag] = TiffTags.DOUBLE else: if py3: if all(isinstance(v, str) for v in values): - self.tagtype[tag] = 2 + self.tagtype[tag] = TiffTags.ASCII else: # Never treat data as binary by default on Python 2. - self.tagtype[tag] = 2 + self.tagtype[tag] = TiffTags.ASCII - if self.tagtype[tag] == 7 and py3: - values = [value.encode("ascii", 'replace') if isinstance( - value, str) else value] - elif self.tagtype[tag] == 5: - values = [float(v) if isinstance(v, int) else v - for v in values] + if self.tagtype[tag] == TiffTags.UNDEFINED and py3: + values = [ + value.encode("ascii", "replace") if isinstance(value, str) else value + ] + elif self.tagtype[tag] == TiffTags.RATIONAL: + values = [float(v) if isinstance(v, int) else v for v in values] values = tuple(info.cvt_enum(value) for value in values) @@ -584,18 +585,23 @@ class ImageFileDirectory_v2(MutableMapping): # Spec'd length == 1, Actual > 1, Warn and truncate. Formerly barfed. # No Spec, Actual length 1, Formerly (<4.2) returned a 1 element tuple. # Don't mess with the legacy api, since it's frozen. - if (info.length == 1) or \ - (info.length is None and len(values) == 1 and not legacy_api): + if (info.length == 1) or ( + info.length is None and len(values) == 1 and not legacy_api + ): # Don't mess with the legacy api, since it's frozen. - if legacy_api and self.tagtype[tag] in [5, 10]: # rationals - values = values, + if legacy_api and self.tagtype[tag] in [ + TiffTags.RATIONAL, + TiffTags.SIGNED_RATIONAL, + ]: # rationals + values = (values,) try: dest[tag], = values except ValueError: # We've got a builtin tag with 1 expected entry warnings.warn( - "Metadata Warning, tag %s had too many entries: %s, expected 1" % ( - tag, len(values))) + "Metadata Warning, tag %s had too many entries: %s, expected 1" + % (tag, len(values)) + ) dest[tag] = values[0] else: @@ -620,36 +626,51 @@ class ImageFileDirectory_v2(MutableMapping): def _register_loader(idx, size): def decorator(func): from .TiffTags import TYPES + if func.__name__.startswith("load_"): TYPES[idx] = func.__name__[5:].replace("_", " ") - _load_dispatch[idx] = size, func + _load_dispatch[idx] = size, func # noqa: F821 return func + return decorator def _register_writer(idx): def decorator(func): - _write_dispatch[idx] = func + _write_dispatch[idx] = func # noqa: F821 return func + return decorator def _register_basic(idx_fmt_name): from .TiffTags import TYPES + idx, fmt, name = idx_fmt_name TYPES[idx] = name size = struct.calcsize("=" + fmt) - _load_dispatch[idx] = size, lambda self, data, legacy_api=True: ( - self._unpack("{}{}".format(len(data) // size, fmt), data)) - _write_dispatch[idx] = lambda self, *values: ( - b"".join(self._pack(fmt, value) for value in values)) + _load_dispatch[idx] = ( # noqa: F821 + size, + lambda self, data, legacy_api=True: ( + self._unpack("{}{}".format(len(data) // size, fmt), data) + ), + ) + _write_dispatch[idx] = lambda self, *values: ( # noqa: F821 + b"".join(self._pack(fmt, value) for value in values) + ) - list(map(_register_basic, - [(3, "H", "short"), - (4, "L", "long"), - (6, "b", "signed byte"), - (8, "h", "signed short"), - (9, "l", "signed long"), - (11, "f", "float"), - (12, "d", "double")])) + list( + map( + _register_basic, + [ + (TiffTags.SHORT, "H", "short"), + (TiffTags.LONG, "L", "long"), + (TiffTags.SIGNED_BYTE, "b", "signed byte"), + (TiffTags.SIGNED_SHORT, "h", "signed short"), + (TiffTags.SIGNED_LONG, "l", "signed long"), + (TiffTags.FLOAT, "f", "float"), + (TiffTags.DOUBLE, "d", "double"), + ], + ) + ) @_register_loader(1, 1) # Basic type, except for the legacy API. def load_byte(self, data, legacy_api=True): @@ -669,21 +690,23 @@ class ImageFileDirectory_v2(MutableMapping): def write_string(self, value): # remerge of https://github.com/python-pillow/Pillow/pull/1416 if sys.version_info.major == 2: - value = value.decode('ascii', 'replace') - return b"" + value.encode('ascii', 'replace') + b"\0" + value = value.decode("ascii", "replace") + return b"" + value.encode("ascii", "replace") + b"\0" @_register_loader(5, 8) def load_rational(self, data, legacy_api=True): vals = self._unpack("{}L".format(len(data) // 4), data) - def combine(a, b): return (a, b) if legacy_api else IFDRational(a, b) - return tuple(combine(num, denom) - for num, denom in zip(vals[::2], vals[1::2])) + def combine(a, b): + return (a, b) if legacy_api else IFDRational(a, b) + + return tuple(combine(num, denom) for num, denom in zip(vals[::2], vals[1::2])) @_register_writer(5) def write_rational(self, *values): - return b"".join(self._pack("2L", *_limit_rational(frac, 2 ** 31)) - for frac in values) + return b"".join( + self._pack("2L", *_limit_rational(frac, 2 ** 31)) for frac in values + ) @_register_loader(7, 1) def load_undefined(self, data, legacy_api=True): @@ -697,21 +720,24 @@ class ImageFileDirectory_v2(MutableMapping): def load_signed_rational(self, data, legacy_api=True): vals = self._unpack("{}l".format(len(data) // 4), data) - def combine(a, b): return (a, b) if legacy_api else IFDRational(a, b) - return tuple(combine(num, denom) - for num, denom in zip(vals[::2], vals[1::2])) + def combine(a, b): + return (a, b) if legacy_api else IFDRational(a, b) + + return tuple(combine(num, denom) for num, denom in zip(vals[::2], vals[1::2])) @_register_writer(10) def write_signed_rational(self, *values): - return b"".join(self._pack("2L", *_limit_rational(frac, 2 ** 30)) - for frac in values) + return b"".join( + self._pack("2L", *_limit_rational(frac, 2 ** 30)) for frac in values + ) def _ensure_read(self, fp, size): ret = fp.read(size) if len(ret) != size: - raise IOError("Corrupt EXIF data. " + - "Expecting to read %d bytes but only got %d. " % - (size, len(ret))) + raise IOError( + "Corrupt EXIF data. " + + "Expecting to read %d bytes but only got %d. " % (size, len(ret)) + ) return ret def load(self, fp): @@ -721,13 +747,14 @@ class ImageFileDirectory_v2(MutableMapping): try: for i in range(self._unpack("H", self._ensure_read(fp, 2))[0]): - tag, typ, count, data = self._unpack("HHL4s", - self._ensure_read(fp, 12)) + tag, typ, count, data = self._unpack("HHL4s", self._ensure_read(fp, 12)) if DEBUG: tagname = TiffTags.lookup(tag).name typname = TYPES.get(typ, "unknown") - print("tag: %s (%d) - type: %s (%d)" % - (tagname, tag, typname, typ), end=" ") + print( + "tag: %s (%d) - type: %s (%d)" % (tagname, tag, typname, typ), + end=" ", + ) try: unit_size, handler = self._load_dispatch[typ] @@ -740,8 +767,10 @@ class ImageFileDirectory_v2(MutableMapping): here = fp.tell() offset, = self._unpack("L", data) if DEBUG: - print("Tag Location: %s - Data Location: %s" % - (here, offset), end=" ") + print( + "Tag Location: %s - Data Location: %s" % (here, offset), + end=" ", + ) fp.seek(offset) data = ImageFile._safe_read(fp, size) fp.seek(here) @@ -749,9 +778,11 @@ class ImageFileDirectory_v2(MutableMapping): data = data[:size] if len(data) != size: - warnings.warn("Possibly corrupt EXIF data. " - "Expecting to read %d bytes but only got %d." - " Skipping tag %s" % (size, len(data), tag)) + warnings.warn( + "Possibly corrupt EXIF data. " + "Expecting to read %d bytes but only got %d." + " Skipping tag %s" % (size, len(data), tag) + ) continue if not data: @@ -771,17 +802,12 @@ class ImageFileDirectory_v2(MutableMapping): warnings.warn(str(msg)) return - def save(self, fp): - - if fp.tell() == 0: # skip TIFF header on subsequent pages - # tiff header -- PIL always starts the first IFD at offset 8 - fp.write(self._prefix + self._pack("HL", 42, 8)) - + def tobytes(self, offset=0): # FIXME What about tagdata? - fp.write(self._pack("H", len(self._tags_v2))) + result = self._pack("H", len(self._tags_v2)) entries = [] - offset = fp.tell() + len(self._tags_v2) * 12 + 4 + offset = offset + len(result) + len(self._tags_v2) * 12 + 4 stripoffsets = None # pass 1: convert tags to binary format @@ -797,29 +823,32 @@ class ImageFileDirectory_v2(MutableMapping): if DEBUG: tagname = TiffTags.lookup(tag).name typname = TYPES.get(typ, "unknown") - print("save: %s (%d) - type: %s (%d)" % - (tagname, tag, typname, typ), end=" ") + print( + "save: %s (%d) - type: %s (%d)" % (tagname, tag, typname, typ), + end=" ", + ) if len(data) >= 16: print("- value: " % len(data)) else: print("- value:", values) # count is sum of lengths for string and arbitrary data - count = len(data) if typ in [2, 7] else len(values) + if typ in [TiffTags.BYTE, TiffTags.ASCII, TiffTags.UNDEFINED]: + count = len(data) + else: + count = len(values) # figure out if data fits into the entry if len(data) <= 4: entries.append((tag, typ, count, data.ljust(4, b"\0"), b"")) else: - entries.append((tag, typ, count, self._pack("L", offset), - data)) + entries.append((tag, typ, count, self._pack("L", offset), data)) offset += (len(data) + 1) // 2 * 2 # pad to word # update strip offset data to point beyond auxiliary data if stripoffsets is not None: tag, typ, count, value, data = entries[stripoffsets] if data: - raise NotImplementedError( - "multistrip support not yet implemented") + raise NotImplementedError("multistrip support not yet implemented") value = self._pack("L", self._unpack("L", value)[0] + offset) entries[stripoffsets] = tag, typ, count, value, data @@ -827,18 +856,29 @@ class ImageFileDirectory_v2(MutableMapping): for tag, typ, count, value, data in entries: if DEBUG > 1: print(tag, typ, count, repr(value), repr(data)) - fp.write(self._pack("HHL4s", tag, typ, count, value)) + result += self._pack("HHL4s", tag, typ, count, value) # -- overwrite here for multi-page -- - fp.write(b"\0\0\0\0") # end of entries + result += b"\0\0\0\0" # end of entries # pass 3: write auxiliary data to file for tag, typ, count, value, data in entries: - fp.write(data) + result += data if len(data) & 1: - fp.write(b"\0") + result += b"\0" - return offset + return result + + def save(self, fp): + + if fp.tell() == 0: # skip TIFF header on subsequent pages + # tiff header -- PIL always starts the first IFD at offset 8 + fp.write(self._prefix + self._pack("HL", 42, 8)) + + offset = fp.tell() + result = self.tobytes(offset) + fp.write(result) + return offset + len(result) ImageFileDirectory_v2._load_dispatch = _load_dispatch @@ -858,7 +898,7 @@ class ImageFileDirectory_v1(ImageFileDirectory_v2): ifd = ImageFileDirectory_v1() ifd[key] = 'Some Data' - ifd.tagtype[key] = 2 + ifd.tagtype[key] = TiffTags.ASCII print(ifd[key]) ('Some Data',) @@ -869,6 +909,7 @@ class ImageFileDirectory_v1(ImageFileDirectory_v2): .. deprecated:: 3.0.0 """ + def __init__(self, *args, **kwargs): ImageFileDirectory_v2.__init__(self, *args, **kwargs) self._legacy_api = True @@ -933,7 +974,7 @@ class ImageFileDirectory_v1(ImageFileDirectory_v2): self._setitem(tag, handler(self, data, legacy), legacy) val = self._tags_v1[tag] if not isinstance(val, (tuple, bytes)): - val = val, + val = (val,) return val @@ -944,6 +985,7 @@ ImageFileDirectory = ImageFileDirectory_v1 ## # Image plugin for TIFF files. + class TiffImageFile(ImageFile.ImageFile): format = "TIFF" @@ -951,7 +993,7 @@ class TiffImageFile(ImageFile.ImageFile): _close_exclusive_fp_after_loading = False def _open(self): - "Open the first image in a TIFF file" + """Open the first image in a TIFF file""" # Header ifh = self.fp.read(8) @@ -968,7 +1010,6 @@ class TiffImageFile(ImageFile.ImageFile): self.__fp = self.fp self._frame_pos = [] self._n_frames = None - self._is_animated = None if DEBUG: print("*** TiffImageFile._open ***") @@ -982,33 +1023,18 @@ class TiffImageFile(ImageFile.ImageFile): def n_frames(self): if self._n_frames is None: current = self.tell() - try: - while True: - self._seek(self.tell() + 1) - except EOFError: - self._n_frames = self.tell() + 1 + self._seek(len(self._frame_pos)) + while self._n_frames is None: + self._seek(self.tell() + 1) self.seek(current) return self._n_frames @property def is_animated(self): - if self._is_animated is None: - if self._n_frames is not None: - self._is_animated = self._n_frames != 1 - else: - current = self.tell() - - try: - self.seek(1) - self._is_animated = True - except EOFError: - self._is_animated = False - - self.seek(current) return self._is_animated def seek(self, frame): - "Select a given frame as current image" + """Select a given frame as current image""" if not self._seek_check(frame): return self._seek(frame) @@ -1024,9 +1050,10 @@ class TiffImageFile(ImageFile.ImageFile): if not self.__next: raise EOFError("no more images in TIFF file") if DEBUG: - print("Seeking to frame %s, on frame %s, " - "__next %s, location: %s" % - (frame, self.__frame, self.__next, self.fp.tell())) + print( + "Seeking to frame %s, on frame %s, __next %s, location: %s" + % (frame, self.__frame, self.__next, self.fp.tell()) + ) # reset python3 buffered io handle in case fp # was passed to libtiff, invalidating the buffer self.fp.tell() @@ -1036,17 +1063,20 @@ class TiffImageFile(ImageFile.ImageFile): print("Loading tags, location: %s" % self.fp.tell()) self.tag_v2.load(self.fp) self.__next = self.tag_v2.next + if self.__next == 0: + self._n_frames = frame + 1 + if len(self._frame_pos) == 1: + self._is_animated = self.__next != 0 self.__frame += 1 self.fp.seek(self._frame_pos[frame]) self.tag_v2.load(self.fp) - self.__next = self.tag_v2.next # fill the legacy tag/ifd entries self.tag = self.ifd = ImageFileDirectory_v1.from_v2(self.tag_v2) self.__frame = frame self._setup() def tell(self): - "Return the current frame number" + """Return the current frame number""" return self.__frame @property @@ -1056,9 +1086,9 @@ class TiffImageFile(ImageFile.ImageFile): @size.setter def size(self, value): warnings.warn( - 'Setting the size of a TIFF image directly is deprecated, and will' - ' be removed in a future version. Use the resize method instead.', - DeprecationWarning + "Setting the size of a TIFF image directly is deprecated, and will" + " be removed in a future version. Use the resize method instead.", + DeprecationWarning, ) self._size = value @@ -1068,9 +1098,23 @@ class TiffImageFile(ImageFile.ImageFile): return super(TiffImageFile, self).load() def load_end(self): + if self._tile_orientation: + method = { + 2: Image.FLIP_LEFT_RIGHT, + 3: Image.ROTATE_180, + 4: Image.FLIP_TOP_BOTTOM, + 5: Image.TRANSPOSE, + 6: Image.ROTATE_270, + 7: Image.TRANSVERSE, + 8: Image.ROTATE_90, + }.get(self._tile_orientation) + if method is not None: + self.im = self.im.transpose(method) + self._size = self.im.size + # allow closing if we're on the first frame, there's no next # This is the ImageFile.load path only, libtiff specific below. - if self.__frame == 0 and not self.__next: + if not self._is_animated: self._close_exclusive_fp_after_loading = True def _load_libtiff(self): @@ -1092,7 +1136,7 @@ class TiffImageFile(ImageFile.ImageFile): # (self._compression, (extents tuple), # 0, (rawmode, self._compression, fp)) extents = self.tile[0][1] - args = list(self.tile[0][3]) + [self.tag_v2.offset] + args = list(self.tile[0][3]) # To be nice on memory footprint, if there's a # file descriptor, use that instead of reading @@ -1113,13 +1157,15 @@ class TiffImageFile(ImageFile.ImageFile): if fp: args[2] = fp - decoder = Image._getdecoder(self.mode, 'libtiff', tuple(args), - self.decoderconfig) + decoder = Image._getdecoder( + self.mode, "libtiff", tuple(args), self.decoderconfig + ) try: decoder.setimage(self.im, extents) except ValueError: raise IOError("Couldn't set the image") + close_self_fp = self._exclusive_fp and not self._is_animated if hasattr(self.fp, "getvalue"): # We've got a stringio like thing passed in. Yay for all in memory. # The decoder needs the entire file in one shot, so there's not @@ -1133,27 +1179,31 @@ class TiffImageFile(ImageFile.ImageFile): if DEBUG: print("have getvalue. just sending in a string from getvalue") n, err = decoder.decode(self.fp.getvalue()) - elif hasattr(self.fp, "fileno"): + elif fp: # we've got a actual file on disk, pass in the fp. if DEBUG: print("have fileno, calling fileno version of the decoder.") - self.fp.seek(0) + if not close_self_fp: + self.fp.seek(0) # 4 bytes, otherwise the trace might error out n, err = decoder.decode(b"fpfp") else: # we have something else. if DEBUG: print("don't have fileno or getvalue. just reading") + self.fp.seek(0) # UNDONE -- so much for that buffer size thing. n, err = decoder.decode(self.fp.read()) self.tile = [] self.readonly = 0 + + self.load_end() + # libtiff closed the fp in a, we need to close self.fp, if possible - if self._exclusive_fp: - if self.__frame == 0 and not self.__next: - self.fp.close() - self.fp = None # might be shared + if close_self_fp: + self.fp.close() + self.fp = None # might be shared if err < 0: raise IOError(err) @@ -1161,7 +1211,7 @@ class TiffImageFile(ImageFile.ImageFile): return Image.Image.load(self) def _setup(self): - "Setup this image object based on current tags" + """Setup this image object based on current tags""" if 0xBC01 in self.tag_v2: raise IOError("Windows Media Photo files not yet supported") @@ -1174,6 +1224,10 @@ class TiffImageFile(ImageFile.ImageFile): # the specification photo = self.tag_v2.get(PHOTOMETRIC_INTERPRETATION, 0) + # old style jpeg compression images most certainly are YCbCr + if self._compression == "tiff_jpeg": + photo = 6 + fillorder = self.tag_v2.get(FILLORDER, 1) if DEBUG: @@ -1185,16 +1239,15 @@ class TiffImageFile(ImageFile.ImageFile): print("- YCbCr subsampling:", self.tag.get(530)) # size - xsize = self.tag_v2.get(IMAGEWIDTH) - ysize = self.tag_v2.get(IMAGELENGTH) + xsize = int(self.tag_v2.get(IMAGEWIDTH)) + ysize = int(self.tag_v2.get(IMAGELENGTH)) self._size = xsize, ysize if DEBUG: print("- size:", self.size) sampleFormat = self.tag_v2.get(SAMPLEFORMAT, (1,)) - if (len(sampleFormat) > 1 - and max(sampleFormat) == min(sampleFormat) == 1): + if len(sampleFormat) > 1 and max(sampleFormat) == min(sampleFormat) == 1: # SAMPLEFORMAT is properly per band, so an RGB image will # be (1,1,1). But, we don't support per band pixel types, # and anything more than one band is a uint8. So, just @@ -1217,8 +1270,14 @@ class TiffImageFile(ImageFile.ImageFile): bps_tuple = bps_tuple * bps_count # mode: check photometric interpretation and bits per pixel - key = (self.tag_v2.prefix, photo, sampleFormat, fillorder, - bps_tuple, extra_tuple) + key = ( + self.tag_v2.prefix, + photo, + sampleFormat, + fillorder, + bps_tuple, + extra_tuple, + ) if DEBUG: print("format key:", key) try: @@ -1240,11 +1299,11 @@ class TiffImageFile(ImageFile.ImageFile): if xres and yres: resunit = self.tag_v2.get(RESOLUTION_UNIT) if resunit == 2: # dots per inch - self.info["dpi"] = xres, yres + self.info["dpi"] = int(xres + 0.5), int(yres + 0.5) elif resunit == 3: # dots per centimeter. convert to dpi - self.info["dpi"] = xres * 2.54, yres * 2.54 + self.info["dpi"] = int(xres * 2.54 + 0.5), int(yres * 2.54 + 0.5) elif resunit is None: # used to default to 1, but now 2) - self.info["dpi"] = xres, yres + self.info["dpi"] = int(xres + 0.5), int(yres + 0.5) # For backward compatibility, # we also preserve the old behavior self.info["resolution"] = xres, yres @@ -1254,7 +1313,7 @@ class TiffImageFile(ImageFile.ImageFile): # build tile descriptors x = y = layer = 0 self.tile = [] - self.use_load_libtiff = READ_LIBTIFF or self._compression != 'raw' + self.use_load_libtiff = READ_LIBTIFF or self._compression != "raw" if self.use_load_libtiff: # Decoder expects entire file as one tile. # There's a buffer size limit in load (64k) @@ -1281,20 +1340,17 @@ class TiffImageFile(ImageFile.ImageFile): # we're expecting image byte order. So, if the rawmode # contains I;16, we need to convert from native to image # byte order. - if rawmode == 'I;16': - rawmode = 'I;16N' - if ';16B' in rawmode: - rawmode = rawmode.replace(';16B', ';16N') - if ';16L' in rawmode: - rawmode = rawmode.replace(';16L', ';16N') + if rawmode == "I;16": + rawmode = "I;16N" + if ";16B" in rawmode: + rawmode = rawmode.replace(";16B", ";16N") + if ";16L" in rawmode: + rawmode = rawmode.replace(";16L", ";16N") # Offset in the tile tuple is 0, we go from 0,0 to # w,h, and we only do this once -- eds - a = (rawmode, self._compression, False) - self.tile.append( - (self._compression, - (0, 0, xsize, ysize), - 0, a)) + a = (rawmode, self._compression, False, self.tag_v2.offset) + self.tile.append(("libtiff", (0, 0, xsize, ysize), 0, a)) elif STRIPOFFSETS in self.tag_v2 or TILEOFFSETS in self.tag_v2: # striped image @@ -1323,9 +1379,13 @@ class TiffImageFile(ImageFile.ImageFile): a = (tile_rawmode, int(stride), 1) self.tile.append( - (self._compression, - (x, y, min(x+w, xsize), min(y+h, ysize)), - offset, a)) + ( + self._compression, + (x, y, min(x + w, xsize), min(y + h, ysize)), + offset, + a, + ) + ) x = x + w if x >= self.size[0]: x, y = 0, y + h @@ -1339,14 +1399,25 @@ class TiffImageFile(ImageFile.ImageFile): # Fix up info. if ICCPROFILE in self.tag_v2: - self.info['icc_profile'] = self.tag_v2[ICCPROFILE] + self.info["icc_profile"] = self.tag_v2[ICCPROFILE] # fixup palette descriptor - if self.mode == "P": + if self.mode in ["P", "PA"]: palette = [o8(b // 256) for b in self.tag_v2[COLORMAP]] self.palette = ImagePalette.raw("RGB;L", b"".join(palette)) + self._tile_orientation = self.tag_v2.get(0x0112) + + def _close__fp(self): + try: + if self.__fp != self.fp: + self.__fp.close() + except AttributeError: + pass + finally: + self.__fp = None + # # -------------------------------------------------------------------- @@ -1373,7 +1444,6 @@ SAVE_INFO = { "CMYK": ("CMYK", II, 5, 1, (8, 8, 8, 8), None), "YCbCr": ("YCbCr", II, 6, 1, (8, 8, 8), None), "LAB": ("LAB", II, 8, 1, (8, 8, 8), None), - "I;32BS": ("I;32BS", MM, 1, 2, (32,), None), "I;16B": ("I;16B", MM, 1, 1, (16,), None), "I;16BS": ("I;16BS", MM, 1, 2, (16,), None), @@ -1390,14 +1460,14 @@ def _save(im, fp, filename): ifd = ImageFileDirectory_v2(prefix=prefix) - compression = im.encoderinfo.get('compression', im.info.get('compression')) + compression = im.encoderinfo.get("compression", im.info.get("compression")) if compression is None: - compression = 'raw' + compression = "raw" - libtiff = WRITE_LIBTIFF or compression != 'raw' + libtiff = WRITE_LIBTIFF or compression != "raw" # required for color libtiff images - ifd[PLANAR_CONFIGURATION] = getattr(im, '_planar_configuration', 1) + ifd[PLANAR_CONFIGURATION] = getattr(im, "_planar_configuration", 1) ifd[IMAGEWIDTH] = im.size[0] ifd[IMAGELENGTH] = im.size[1] @@ -1412,15 +1482,21 @@ def _save(im, fp, filename): ifd[key] = info.get(key) try: ifd.tagtype[key] = info.tagtype[key] - except: - pass # might not be an IFD, Might not have populated type + except Exception: + pass # might not be an IFD. Might not have populated type # additions written by Greg Couch, gregc@cgl.ucsf.edu # inspired by image-sig posting from Kevin Cazabon, kcazabon@home.com - if hasattr(im, 'tag_v2'): + if hasattr(im, "tag_v2"): # preserve tags from original TIFF image file - for key in (RESOLUTION_UNIT, X_RESOLUTION, Y_RESOLUTION, - IPTC_NAA_CHUNK, PHOTOSHOP_CHUNK, XMP): + for key in ( + RESOLUTION_UNIT, + X_RESOLUTION, + Y_RESOLUTION, + IPTC_NAA_CHUNK, + PHOTOSHOP_CHUNK, + XMP, + ): if key in im.tag_v2: ifd[key] = im.tag_v2[key] ifd.tagtype[key] = im.tag_v2.tagtype[key] @@ -1430,24 +1506,26 @@ def _save(im, fp, filename): if "icc_profile" in im.info: ifd[ICCPROFILE] = im.info["icc_profile"] - for key, name in [(IMAGEDESCRIPTION, "description"), - (X_RESOLUTION, "resolution"), - (Y_RESOLUTION, "resolution"), - (X_RESOLUTION, "x_resolution"), - (Y_RESOLUTION, "y_resolution"), - (RESOLUTION_UNIT, "resolution_unit"), - (SOFTWARE, "software"), - (DATE_TIME, "date_time"), - (ARTIST, "artist"), - (COPYRIGHT, "copyright")]: + for key, name in [ + (IMAGEDESCRIPTION, "description"), + (X_RESOLUTION, "resolution"), + (Y_RESOLUTION, "resolution"), + (X_RESOLUTION, "x_resolution"), + (Y_RESOLUTION, "y_resolution"), + (RESOLUTION_UNIT, "resolution_unit"), + (SOFTWARE, "software"), + (DATE_TIME, "date_time"), + (ARTIST, "artist"), + (COPYRIGHT, "copyright"), + ]: if name in im.encoderinfo: ifd[key] = im.encoderinfo[name] dpi = im.encoderinfo.get("dpi") if dpi: ifd[RESOLUTION_UNIT] = 2 - ifd[X_RESOLUTION] = dpi[0] - ifd[Y_RESOLUTION] = dpi[1] + ifd[X_RESOLUTION] = int(dpi[0] + 0.5) + ifd[Y_RESOLUTION] = int(dpi[1] + 0.5) if bits != (1,): ifd[BITSPERSAMPLE] = bits @@ -1460,11 +1538,11 @@ def _save(im, fp, filename): ifd[PHOTOMETRIC_INTERPRETATION] = photo - if im.mode == "P": + if im.mode in ["P", "PA"]: lut = im.im.getpalette("RGB", "RGB;L") ifd[COLORMAP] = tuple(i8(v) * 256 for v in lut) # data orientation - stride = len(bits) * ((im.size[0]*bits[0]+7)//8) + stride = len(bits) * ((im.size[0] * bits[0] + 7) // 8) ifd[ROWSPERSTRIP] = im.size[1] ifd[STRIPBYTECOUNTS] = stride * im.size[1] ifd[STRIPOFFSETS] = 0 # this is adjusted by IFD writer @@ -1472,6 +1550,16 @@ def _save(im, fp, filename): ifd[COMPRESSION] = COMPRESSION_INFO_REV.get(compression, 1) if libtiff: + if "quality" in im.encoderinfo: + quality = im.encoderinfo["quality"] + if not isinstance(quality, int) or quality < 0 or quality > 100: + raise ValueError("Invalid quality setting") + if compression != "jpeg": + raise ValueError( + "quality setting only supported for 'jpeg' compression" + ) + ifd[JPEGQUALITY] = quality + if DEBUG: print("Saving using libtiff encoder") print("Items: %s" % sorted(ifd.items())) @@ -1483,9 +1571,24 @@ def _save(im, fp, filename): except io.UnsupportedOperation: pass + # optional types for non core tags + types = {} + # SAMPLEFORMAT is determined by the image format and should not be copied + # from legacy_ifd. # STRIPOFFSETS and STRIPBYTECOUNTS are added by the library # based on the data in the strip. - blocklist = [STRIPOFFSETS, STRIPBYTECOUNTS] + # The other tags expect arrays with a certain length (fixed or depending on + # BITSPERSAMPLE, etc), passing arrays with a different length will result in + # segfaults. Block these tags until we add extra validation. + blocklist = [ + COLORMAP, + REFERENCEBLACKWHITE, + SAMPLEFORMAT, + STRIPBYTECOUNTS, + STRIPOFFSETS, + TRANSFERFUNCTION, + ] + atts = {} # bits per sample is a single short in the tiff directory, not a list. atts[BITSPERSAMPLE] = bits[0] @@ -1493,21 +1596,33 @@ def _save(im, fp, filename): # the original file, e.g x,y resolution so that we can # save(load('')) == original file. legacy_ifd = {} - if hasattr(im, 'tag'): + if hasattr(im, "tag"): legacy_ifd = im.tag.to_v2() - for tag, value in itertools.chain(ifd.items(), - getattr(im, 'tag_v2', {}).items(), - legacy_ifd.items()): + for tag, value in itertools.chain( + ifd.items(), getattr(im, "tag_v2", {}).items(), legacy_ifd.items() + ): # Libtiff can only process certain core items without adding - # them to the custom dictionary. It will segfault if it attempts - # to add a custom tag without the dictionary entry - # - # UNDONE -- add code for the custom dictionary + # them to the custom dictionary. + # Custom items are supported for int, float, unicode, string and byte + # values. Other types and tuples require a tagtype. if tag not in TiffTags.LIBTIFF_CORE: - continue + if TiffTags.lookup(tag).type == TiffTags.UNDEFINED: + continue + if distutils.version.StrictVersion( + _libtiff_version() + ) < distutils.version.StrictVersion("4.0"): + continue + + if tag in ifd.tagtype: + types[tag] = ifd.tagtype[tag] + elif not ( + isinstance(value, (int, float, str, bytes)) + or (not py3 and isinstance(value, unicode)) # noqa: F821 + ): + continue if tag not in atts and tag not in blocklist: - if isinstance(value, str if py3 else unicode): - atts[tag] = value.encode('ascii', 'replace') + b"\0" + if isinstance(value, str if py3 else unicode): # noqa: F821 + atts[tag] = value.encode("ascii", "replace") + b"\0" elif isinstance(value, IFDRational): atts[tag] = float(value) else: @@ -1520,15 +1635,20 @@ def _save(im, fp, filename): # we're storing image byte order. So, if the rawmode # contains I;16, we need to convert from native to image # byte order. - if im.mode in ('I;16B', 'I;16'): - rawmode = 'I;16N' + if im.mode in ("I;16B", "I;16"): + rawmode = "I;16N" - a = (rawmode, compression, _fp, filename, atts) - e = Image._getencoder(im.mode, 'libtiff', a, im.encoderconfig) - e.setimage(im.im, (0, 0)+im.size) + # Pass tags as sorted list so that the tags are set in a fixed order. + # This is required by libtiff for some tags. For example, the JPEGQUALITY + # pseudo tag requires that the COMPRESS tag was already set. + tags = list(atts.items()) + tags.sort() + a = (rawmode, compression, _fp, filename, tags, types) + e = Image._getencoder(im.mode, "libtiff", a, im.encoderconfig) + e.setimage(im.im, (0, 0) + im.size) while True: # undone, change to self.decodermaxblock: - l, s, d = e.encode(16*1024) + l, s, d = e.encode(16 * 1024) if not _fp: fp.write(d) if s: @@ -1539,9 +1659,9 @@ def _save(im, fp, filename): else: offset = ifd.save(fp) - ImageFile._save(im, fp, [ - ("raw", (0, 0)+im.size, offset, (rawmode, stride, 1)) - ]) + ImageFile._save( + im, fp, [("raw", (0, 0) + im.size, offset, (rawmode, stride, 1))] + ) # -- helper for multi-page save -- if "_debug_multipage" in im.encoderinfo: @@ -1575,7 +1695,7 @@ class AppendingTiffWriter: Tags = {273, 288, 324, 519, 520, 521} def __init__(self, fn, new=False): - if hasattr(fn, 'read'): + if hasattr(fn, "read"): self.f = fn self.close_fp = False else: @@ -1626,8 +1746,7 @@ class AppendingTiffWriter: return if IIMM != self.IIMM: - raise RuntimeError("IIMM of new page doesn't match IIMM of " - "first page") + raise RuntimeError("IIMM of new page doesn't match IIMM of first page") IFDoffset = self.readLong() IFDoffset += self.offsetOfNewPage @@ -1652,7 +1771,7 @@ class AppendingTiffWriter: def tell(self): return self.f.tell() - self.offsetOfNewPage - def seek(self, offset, whence): + def seek(self, offset, whence=io.SEEK_SET): if whence == os.SEEK_SET: offset += self.offsetOfNewPage @@ -1701,34 +1820,29 @@ class AppendingTiffWriter: self.f.seek(-2, os.SEEK_CUR) bytesWritten = self.f.write(struct.pack(self.longFmt, value)) if bytesWritten is not None and bytesWritten != 4: - raise RuntimeError("wrote only %u bytes but wanted 4" % - bytesWritten) + raise RuntimeError("wrote only %u bytes but wanted 4" % bytesWritten) def rewriteLastShort(self, value): self.f.seek(-2, os.SEEK_CUR) bytesWritten = self.f.write(struct.pack(self.shortFmt, value)) if bytesWritten is not None and bytesWritten != 2: - raise RuntimeError("wrote only %u bytes but wanted 2" % - bytesWritten) + raise RuntimeError("wrote only %u bytes but wanted 2" % bytesWritten) def rewriteLastLong(self, value): self.f.seek(-4, os.SEEK_CUR) bytesWritten = self.f.write(struct.pack(self.longFmt, value)) if bytesWritten is not None and bytesWritten != 4: - raise RuntimeError("wrote only %u bytes but wanted 4" % - bytesWritten) + raise RuntimeError("wrote only %u bytes but wanted 4" % bytesWritten) def writeShort(self, value): bytesWritten = self.f.write(struct.pack(self.shortFmt, value)) if bytesWritten is not None and bytesWritten != 2: - raise RuntimeError("wrote only %u bytes but wanted 2" % - bytesWritten) + raise RuntimeError("wrote only %u bytes but wanted 2" % bytesWritten) def writeLong(self, value): bytesWritten = self.f.write(struct.pack(self.longFmt, value)) if bytesWritten is not None and bytesWritten != 4: - raise RuntimeError("wrote only %u bytes but wanted 4" % - bytesWritten) + raise RuntimeError("wrote only %u bytes but wanted 4" % bytesWritten) def close(self): self.finalize() @@ -1738,12 +1852,11 @@ class AppendingTiffWriter: numTags = self.readShort() for i in range(numTags): - tag, fieldType, count = struct.unpack(self.tagFormat, - self.f.read(8)) + tag, fieldType, count = struct.unpack(self.tagFormat, self.f.read(8)) fieldSize = self.fieldSizes[fieldType] totalSize = fieldSize * count - isLocal = (totalSize <= 4) + isLocal = totalSize <= 4 if not isLocal: offset = self.readLong() offset += self.offsetOfNewPage @@ -1753,13 +1866,15 @@ class AppendingTiffWriter: curPos = self.f.tell() if isLocal: - self.fixOffsets(count, isShort=(fieldSize == 2), - isLong=(fieldSize == 4)) + self.fixOffsets( + count, isShort=(fieldSize == 2), isLong=(fieldSize == 4) + ) self.f.seek(curPos + 4) else: self.f.seek(offset) - self.fixOffsets(count, isShort=(fieldSize == 2), - isLong=(fieldSize == 4)) + self.fixOffsets( + count, isShort=(fieldSize == 2), isLong=(fieldSize == 4) + ) self.f.seek(curPos) offset = curPos = None @@ -1784,7 +1899,7 @@ class AppendingTiffWriter: # local (not referenced with another offset) self.rewriteLastShortToLong(offset) self.f.seek(-10, os.SEEK_CUR) - self.writeShort(4) # rewrite the type to LONG + self.writeShort(TiffTags.LONG) # rewrite the type to LONG self.f.seek(8, os.SEEK_CUR) elif isShort: self.rewriteLastShort(offset) @@ -1802,7 +1917,7 @@ def _save_all(im, fp, filename): cur_idx = im.tell() try: with AppendingTiffWriter(fp) as tf: - for ims in [im]+append_images: + for ims in [im] + append_images: ims.encoderinfo = encoderinfo ims.encoderconfig = encoderconfig if not hasattr(ims, "n_frames"): diff --git a/server/www/packages/packages-linux/x64/PIL/TiffTags.py b/server/www/packages/packages-linux/x64/PIL/TiffTags.py index c1e14af..82719db 100644 --- a/server/www/packages/packages-linux/x64/PIL/TiffTags.py +++ b/server/www/packages/packages-linux/x64/PIL/TiffTags.py @@ -23,13 +23,14 @@ from collections import namedtuple class TagInfo(namedtuple("_TagInfo", "value name type length enum")): __slots__ = [] - def __new__(cls, value=None, name="unknown", - type=None, length=None, enum=None): - return super(TagInfo, cls).__new__( - cls, value, name, type, length, enum or {}) + def __new__(cls, value=None, name="unknown", type=None, length=None, enum=None): + return super(TagInfo, cls).__new__(cls, value, name, type, length, enum or {}) def cvt_enum(self, value): - return self.enum.get(value, value) + # Using get will call hash(value), which can be expensive + # for some types (e.g. Fraction). Since self.enum is rarely + # used, it's usually better to test it first. + return self.enum.get(value, value) if self.enum else value def lookup(tag): @@ -41,7 +42,7 @@ def lookup(tag): """ - return TAGS_V2.get(tag, TagInfo(tag, TAGS.get(tag, 'unknown'))) + return TAGS_V2.get(tag, TagInfo(tag, TAGS.get(tag, "unknown"))) ## @@ -61,32 +62,56 @@ ASCII = 2 SHORT = 3 LONG = 4 RATIONAL = 5 +SIGNED_BYTE = 6 UNDEFINED = 7 +SIGNED_SHORT = 8 +SIGNED_LONG = 9 SIGNED_RATIONAL = 10 +FLOAT = 11 DOUBLE = 12 TAGS_V2 = { - 254: ("NewSubfileType", LONG, 1), 255: ("SubfileType", SHORT, 1), 256: ("ImageWidth", LONG, 1), 257: ("ImageLength", LONG, 1), 258: ("BitsPerSample", SHORT, 0), - 259: ("Compression", SHORT, 1, - {"Uncompressed": 1, "CCITT 1d": 2, "Group 3 Fax": 3, - "Group 4 Fax": 4, "LZW": 5, "JPEG": 6, "PackBits": 32773}), - - 262: ("PhotometricInterpretation", SHORT, 1, - {"WhiteIsZero": 0, "BlackIsZero": 1, "RGB": 2, "RGB Palette": 3, - "Transparency Mask": 4, "CMYK": 5, "YCbCr": 6, "CieLAB": 8, - "CFA": 32803, # TIFF/EP, Adobe DNG - "LinearRaw": 32892}), # Adobe DNG + 259: ( + "Compression", + SHORT, + 1, + { + "Uncompressed": 1, + "CCITT 1d": 2, + "Group 3 Fax": 3, + "Group 4 Fax": 4, + "LZW": 5, + "JPEG": 6, + "PackBits": 32773, + }, + ), + 262: ( + "PhotometricInterpretation", + SHORT, + 1, + { + "WhiteIsZero": 0, + "BlackIsZero": 1, + "RGB": 2, + "RGB Palette": 3, + "Transparency Mask": 4, + "CMYK": 5, + "YCbCr": 6, + "CieLAB": 8, + "CFA": 32803, # TIFF/EP, Adobe DNG + "LinearRaw": 32892, # Adobe DNG + }, + ), 263: ("Threshholding", SHORT, 1), 264: ("CellWidth", SHORT, 1), 265: ("CellLength", SHORT, 1), 266: ("FillOrder", SHORT, 1), 269: ("DocumentName", ASCII, 1), - 270: ("ImageDescription", ASCII, 1), 271: ("Make", ASCII, 1), 272: ("Model", ASCII, 1), @@ -95,7 +120,6 @@ TAGS_V2 = { 277: ("SamplesPerPixel", SHORT, 1), 278: ("RowsPerStrip", LONG, 1), 279: ("StripByteCounts", LONG, 0), - 280: ("MinSampleValue", LONG, 0), 281: ("MaxSampleValue", SHORT, 0), 282: ("XResolution", RATIONAL, 1), @@ -106,31 +130,26 @@ TAGS_V2 = { 287: ("YPosition", RATIONAL, 1), 288: ("FreeOffsets", LONG, 1), 289: ("FreeByteCounts", LONG, 1), - 290: ("GrayResponseUnit", SHORT, 1), 291: ("GrayResponseCurve", SHORT, 0), 292: ("T4Options", LONG, 1), 293: ("T6Options", LONG, 1), 296: ("ResolutionUnit", SHORT, 1, {"none": 1, "inch": 2, "cm": 3}), 297: ("PageNumber", SHORT, 2), - 301: ("TransferFunction", SHORT, 0), 305: ("Software", ASCII, 1), 306: ("DateTime", ASCII, 1), - 315: ("Artist", ASCII, 1), 316: ("HostComputer", ASCII, 1), 317: ("Predictor", SHORT, 1, {"none": 1, "Horizontal Differencing": 2}), 318: ("WhitePoint", RATIONAL, 2), 319: ("PrimaryChromaticities", RATIONAL, 6), - 320: ("ColorMap", SHORT, 0), 321: ("HalftoneHints", SHORT, 2), 322: ("TileWidth", LONG, 1), 323: ("TileLength", LONG, 1), 324: ("TileOffsets", LONG, 0), 325: ("TileByteCounts", LONG, 0), - 332: ("InkSet", SHORT, 1), 333: ("InkNames", ASCII, 1), 334: ("NumberOfInks", SHORT, 1), @@ -138,13 +157,10 @@ TAGS_V2 = { 337: ("TargetPrinter", ASCII, 1), 338: ("ExtraSamples", SHORT, 0), 339: ("SampleFormat", SHORT, 0), - 340: ("SMinSampleValue", DOUBLE, 0), 341: ("SMaxSampleValue", DOUBLE, 0), 342: ("TransferRange", SHORT, 6), - 347: ("JPEGTables", UNDEFINED, 1), - # obsolete JPEG tags 512: ("JPEGProc", SHORT, 1), 513: ("JPEGInterchangeFormat", LONG, 1), @@ -155,22 +171,18 @@ TAGS_V2 = { 519: ("JPEGQTables", LONG, 0), 520: ("JPEGDCTables", LONG, 0), 521: ("JPEGACTables", LONG, 0), - 529: ("YCbCrCoefficients", RATIONAL, 3), 530: ("YCbCrSubSampling", SHORT, 2), 531: ("YCbCrPositioning", SHORT, 1), 532: ("ReferenceBlackWhite", RATIONAL, 6), - - 700: ('XMP', BYTE, 1), - + 700: ("XMP", BYTE, 0), 33432: ("Copyright", ASCII, 1), - 34377: ('PhotoshopInfo', BYTE, 1), - + 33723: ("IptcNaaInfo", UNDEFINED, 0), + 34377: ("PhotoshopInfo", BYTE, 0), # FIXME add more tags here - 34665: ("ExifIFD", SHORT, 1), - 34675: ('ICCProfile', UNDEFINED, 1), - 34853: ('GPSInfoIFD', BYTE, 1), - + 34665: ("ExifIFD", LONG, 1), + 34675: ("ICCProfile", UNDEFINED, 1), + 34853: ("GPSInfoIFD", BYTE, 1), # MPInfo 45056: ("MPFVersion", UNDEFINED, 1), 45057: ("NumberOfImages", LONG, 1), @@ -191,159 +203,157 @@ TAGS_V2 = { 45579: ("YawAngle", SIGNED_RATIONAL, 1), 45580: ("PitchAngle", SIGNED_RATIONAL, 1), 45581: ("RollAngle", SIGNED_RATIONAL, 1), - 50741: ("MakerNoteSafety", SHORT, 1, {"Unsafe": 0, "Safe": 1}), 50780: ("BestQualityScale", RATIONAL, 1), 50838: ("ImageJMetaDataByteCounts", LONG, 0), # Can be more than one - 50839: ("ImageJMetaData", UNDEFINED, 1) # see Issue #2006 + 50839: ("ImageJMetaData", UNDEFINED, 1), # see Issue #2006 } # Legacy Tags structure # these tags aren't included above, but were in the previous versions -TAGS = {347: 'JPEGTables', - 700: 'XMP', - - # Additional Exif Info - 32932: 'Wang Annotation', - 33434: 'ExposureTime', - 33437: 'FNumber', - 33445: 'MD FileTag', - 33446: 'MD ScalePixel', - 33447: 'MD ColorTable', - 33448: 'MD LabName', - 33449: 'MD SampleInfo', - 33450: 'MD PrepDate', - 33451: 'MD PrepTime', - 33452: 'MD FileUnits', - 33550: 'ModelPixelScaleTag', - 33723: 'IptcNaaInfo', - 33918: 'INGR Packet Data Tag', - 33919: 'INGR Flag Registers', - 33920: 'IrasB Transformation Matrix', - 33922: 'ModelTiepointTag', - 34264: 'ModelTransformationTag', - 34377: 'PhotoshopInfo', - 34735: 'GeoKeyDirectoryTag', - 34736: 'GeoDoubleParamsTag', - 34737: 'GeoAsciiParamsTag', - 34850: 'ExposureProgram', - 34852: 'SpectralSensitivity', - 34855: 'ISOSpeedRatings', - 34856: 'OECF', - 34864: 'SensitivityType', - 34865: 'StandardOutputSensitivity', - 34866: 'RecommendedExposureIndex', - 34867: 'ISOSpeed', - 34868: 'ISOSpeedLatitudeyyy', - 34869: 'ISOSpeedLatitudezzz', - 34908: 'HylaFAX FaxRecvParams', - 34909: 'HylaFAX FaxSubAddress', - 34910: 'HylaFAX FaxRecvTime', - 36864: 'ExifVersion', - 36867: 'DateTimeOriginal', - 36868: 'DateTImeDigitized', - 37121: 'ComponentsConfiguration', - 37122: 'CompressedBitsPerPixel', - 37724: 'ImageSourceData', - 37377: 'ShutterSpeedValue', - 37378: 'ApertureValue', - 37379: 'BrightnessValue', - 37380: 'ExposureBiasValue', - 37381: 'MaxApertureValue', - 37382: 'SubjectDistance', - 37383: 'MeteringMode', - 37384: 'LightSource', - 37385: 'Flash', - 37386: 'FocalLength', - 37396: 'SubjectArea', - 37500: 'MakerNote', - 37510: 'UserComment', - 37520: 'SubSec', - 37521: 'SubSecTimeOriginal', - 37522: 'SubsecTimeDigitized', - 40960: 'FlashPixVersion', - 40961: 'ColorSpace', - 40962: 'PixelXDimension', - 40963: 'PixelYDimension', - 40964: 'RelatedSoundFile', - 40965: 'InteroperabilityIFD', - 41483: 'FlashEnergy', - 41484: 'SpatialFrequencyResponse', - 41486: 'FocalPlaneXResolution', - 41487: 'FocalPlaneYResolution', - 41488: 'FocalPlaneResolutionUnit', - 41492: 'SubjectLocation', - 41493: 'ExposureIndex', - 41495: 'SensingMethod', - 41728: 'FileSource', - 41729: 'SceneType', - 41730: 'CFAPattern', - 41985: 'CustomRendered', - 41986: 'ExposureMode', - 41987: 'WhiteBalance', - 41988: 'DigitalZoomRatio', - 41989: 'FocalLengthIn35mmFilm', - 41990: 'SceneCaptureType', - 41991: 'GainControl', - 41992: 'Contrast', - 41993: 'Saturation', - 41994: 'Sharpness', - 41995: 'DeviceSettingDescription', - 41996: 'SubjectDistanceRange', - 42016: 'ImageUniqueID', - 42032: 'CameraOwnerName', - 42033: 'BodySerialNumber', - 42034: 'LensSpecification', - 42035: 'LensMake', - 42036: 'LensModel', - 42037: 'LensSerialNumber', - 42112: 'GDAL_METADATA', - 42113: 'GDAL_NODATA', - 42240: 'Gamma', - 50215: 'Oce Scanjob Description', - 50216: 'Oce Application Selector', - 50217: 'Oce Identification Number', - 50218: 'Oce ImageLogic Characteristics', - - # Adobe DNG - 50706: 'DNGVersion', - 50707: 'DNGBackwardVersion', - 50708: 'UniqueCameraModel', - 50709: 'LocalizedCameraModel', - 50710: 'CFAPlaneColor', - 50711: 'CFALayout', - 50712: 'LinearizationTable', - 50713: 'BlackLevelRepeatDim', - 50714: 'BlackLevel', - 50715: 'BlackLevelDeltaH', - 50716: 'BlackLevelDeltaV', - 50717: 'WhiteLevel', - 50718: 'DefaultScale', - 50719: 'DefaultCropOrigin', - 50720: 'DefaultCropSize', - 50721: 'ColorMatrix1', - 50722: 'ColorMatrix2', - 50723: 'CameraCalibration1', - 50724: 'CameraCalibration2', - 50725: 'ReductionMatrix1', - 50726: 'ReductionMatrix2', - 50727: 'AnalogBalance', - 50728: 'AsShotNeutral', - 50729: 'AsShotWhiteXY', - 50730: 'BaselineExposure', - 50731: 'BaselineNoise', - 50732: 'BaselineSharpness', - 50733: 'BayerGreenSplit', - 50734: 'LinearResponseLimit', - 50735: 'CameraSerialNumber', - 50736: 'LensInfo', - 50737: 'ChromaBlurRadius', - 50738: 'AntiAliasStrength', - 50740: 'DNGPrivateData', - 50778: 'CalibrationIlluminant1', - 50779: 'CalibrationIlluminant2', - 50784: 'Alias Layer Metadata' - } +TAGS = { + 347: "JPEGTables", + 700: "XMP", + # Additional Exif Info + 32932: "Wang Annotation", + 33434: "ExposureTime", + 33437: "FNumber", + 33445: "MD FileTag", + 33446: "MD ScalePixel", + 33447: "MD ColorTable", + 33448: "MD LabName", + 33449: "MD SampleInfo", + 33450: "MD PrepDate", + 33451: "MD PrepTime", + 33452: "MD FileUnits", + 33550: "ModelPixelScaleTag", + 33723: "IptcNaaInfo", + 33918: "INGR Packet Data Tag", + 33919: "INGR Flag Registers", + 33920: "IrasB Transformation Matrix", + 33922: "ModelTiepointTag", + 34264: "ModelTransformationTag", + 34377: "PhotoshopInfo", + 34735: "GeoKeyDirectoryTag", + 34736: "GeoDoubleParamsTag", + 34737: "GeoAsciiParamsTag", + 34850: "ExposureProgram", + 34852: "SpectralSensitivity", + 34855: "ISOSpeedRatings", + 34856: "OECF", + 34864: "SensitivityType", + 34865: "StandardOutputSensitivity", + 34866: "RecommendedExposureIndex", + 34867: "ISOSpeed", + 34868: "ISOSpeedLatitudeyyy", + 34869: "ISOSpeedLatitudezzz", + 34908: "HylaFAX FaxRecvParams", + 34909: "HylaFAX FaxSubAddress", + 34910: "HylaFAX FaxRecvTime", + 36864: "ExifVersion", + 36867: "DateTimeOriginal", + 36868: "DateTImeDigitized", + 37121: "ComponentsConfiguration", + 37122: "CompressedBitsPerPixel", + 37724: "ImageSourceData", + 37377: "ShutterSpeedValue", + 37378: "ApertureValue", + 37379: "BrightnessValue", + 37380: "ExposureBiasValue", + 37381: "MaxApertureValue", + 37382: "SubjectDistance", + 37383: "MeteringMode", + 37384: "LightSource", + 37385: "Flash", + 37386: "FocalLength", + 37396: "SubjectArea", + 37500: "MakerNote", + 37510: "UserComment", + 37520: "SubSec", + 37521: "SubSecTimeOriginal", + 37522: "SubsecTimeDigitized", + 40960: "FlashPixVersion", + 40961: "ColorSpace", + 40962: "PixelXDimension", + 40963: "PixelYDimension", + 40964: "RelatedSoundFile", + 40965: "InteroperabilityIFD", + 41483: "FlashEnergy", + 41484: "SpatialFrequencyResponse", + 41486: "FocalPlaneXResolution", + 41487: "FocalPlaneYResolution", + 41488: "FocalPlaneResolutionUnit", + 41492: "SubjectLocation", + 41493: "ExposureIndex", + 41495: "SensingMethod", + 41728: "FileSource", + 41729: "SceneType", + 41730: "CFAPattern", + 41985: "CustomRendered", + 41986: "ExposureMode", + 41987: "WhiteBalance", + 41988: "DigitalZoomRatio", + 41989: "FocalLengthIn35mmFilm", + 41990: "SceneCaptureType", + 41991: "GainControl", + 41992: "Contrast", + 41993: "Saturation", + 41994: "Sharpness", + 41995: "DeviceSettingDescription", + 41996: "SubjectDistanceRange", + 42016: "ImageUniqueID", + 42032: "CameraOwnerName", + 42033: "BodySerialNumber", + 42034: "LensSpecification", + 42035: "LensMake", + 42036: "LensModel", + 42037: "LensSerialNumber", + 42112: "GDAL_METADATA", + 42113: "GDAL_NODATA", + 42240: "Gamma", + 50215: "Oce Scanjob Description", + 50216: "Oce Application Selector", + 50217: "Oce Identification Number", + 50218: "Oce ImageLogic Characteristics", + # Adobe DNG + 50706: "DNGVersion", + 50707: "DNGBackwardVersion", + 50708: "UniqueCameraModel", + 50709: "LocalizedCameraModel", + 50710: "CFAPlaneColor", + 50711: "CFALayout", + 50712: "LinearizationTable", + 50713: "BlackLevelRepeatDim", + 50714: "BlackLevel", + 50715: "BlackLevelDeltaH", + 50716: "BlackLevelDeltaV", + 50717: "WhiteLevel", + 50718: "DefaultScale", + 50719: "DefaultCropOrigin", + 50720: "DefaultCropSize", + 50721: "ColorMatrix1", + 50722: "ColorMatrix2", + 50723: "CameraCalibration1", + 50724: "CameraCalibration2", + 50725: "ReductionMatrix1", + 50726: "ReductionMatrix2", + 50727: "AnalogBalance", + 50728: "AsShotNeutral", + 50729: "AsShotWhiteXY", + 50730: "BaselineExposure", + 50731: "BaselineNoise", + 50732: "BaselineSharpness", + 50733: "BayerGreenSplit", + 50734: "LinearResponseLimit", + 50735: "CameraSerialNumber", + 50736: "LensInfo", + 50737: "ChromaBlurRadius", + 50738: "AntiAliasStrength", + 50740: "DNGPrivateData", + 50778: "CalibrationIlluminant1", + 50779: "CalibrationIlluminant2", + 50784: "Alias Layer Metadata", +} def _populate(): @@ -423,15 +433,55 @@ TYPES = {} # 389: case TIFFTAG_REFERENCEBLACKWHITE: # 393: case TIFFTAG_INKNAMES: +# Following pseudo-tags are also handled by default in libtiff: +# TIFFTAG_JPEGQUALITY 65537 + # some of these are not in our TAGS_V2 dict and were included from tiff.h -LIBTIFF_CORE = {255, 256, 257, 258, 259, 262, 263, 266, 274, 277, - 278, 280, 281, 340, 341, 282, 283, 284, 286, 287, - 296, 297, 321, 320, 338, 32995, 322, 323, 32998, - 32996, 339, 32997, 330, 531, 530, 301, 532, 333, - # as above - 269 # this has been in our tests forever, and works - } +# This list also exists in encode.c +LIBTIFF_CORE = { + 255, + 256, + 257, + 258, + 259, + 262, + 263, + 266, + 274, + 277, + 278, + 280, + 281, + 340, + 341, + 282, + 283, + 284, + 286, + 287, + 296, + 297, + 321, + 320, + 338, + 32995, + 322, + 323, + 32998, + 32996, + 339, + 32997, + 330, + 531, + 530, + 301, + 532, + 333, + # as above + 269, # this has been in our tests forever, and works + 65537, +} LIBTIFF_CORE.remove(320) # Array of short, crashes LIBTIFF_CORE.remove(301) # Array of short, crashes diff --git a/server/www/packages/packages-linux/x64/PIL/WalImageFile.py b/server/www/packages/packages-linux/x64/PIL/WalImageFile.py index 6602cc8..e2e1cd4 100644 --- a/server/www/packages/packages-linux/x64/PIL/WalImageFile.py +++ b/server/www/packages/packages-linux/x64/PIL/WalImageFile.py @@ -28,6 +28,7 @@ try: import builtins except ImportError: import __builtin__ + builtins = __builtin__ @@ -46,7 +47,7 @@ def open(filename): def imopen(fp): # read header fields - header = fp.read(32+24+32+12) + header = fp.read(32 + 24 + 32 + 12) size = i32(header, 32), i32(header, 36) offset = i32(header, 40) @@ -62,7 +63,7 @@ def open(filename): # strings are null-terminated im.info["name"] = header[:32].split(b"\0", 1)[0] - next_name = header[56:56+32].split(b"\0", 1)[0] + next_name = header[56 : 56 + 32].split(b"\0", 1)[0] if next_name: im.info["next_name"] = next_name diff --git a/server/www/packages/packages-linux/x64/PIL/WebPImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/WebPImagePlugin.py index e6485c2..18eda6d 100644 --- a/server/www/packages/packages-linux/x64/PIL/WebPImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/WebPImagePlugin.py @@ -1,28 +1,24 @@ -from . import Image, ImageFile -try: - from . import _webp - SUPPORTED = True -except ImportError as e: - SUPPORTED = False from io import BytesIO +from . import Image, ImageFile -_VALID_WEBP_MODES = { - "RGBX": True, - "RGBA": True, - "RGB": True, - } +try: + from . import _webp -_VALID_WEBP_LEGACY_MODES = { - "RGB": True, - "RGBA": True, - } + SUPPORTED = True +except ImportError: + SUPPORTED = False + + +_VALID_WEBP_MODES = {"RGBX": True, "RGBA": True, "RGB": True} + +_VALID_WEBP_LEGACY_MODES = {"RGB": True, "RGBA": True} _VP8_MODES_BY_IDENTIFIER = { b"VP8 ": "RGB", b"VP8X": "RGBA", b"VP8L": "RGBA", # lossless - } +} def _accept(prefix): @@ -32,7 +28,9 @@ def _accept(prefix): if is_riff_file_format and is_webp_file and is_valid_vp8_mode: if not SUPPORTED: - return "image file could not be identified because WEBP support not installed" + return ( + "image file could not be identified because WEBP support not installed" + ) return True @@ -44,8 +42,9 @@ class WebPImageFile(ImageFile.ImageFile): def _open(self): if not _webp.HAVE_WEBPANIM: # Legacy mode - data, width, height, self.mode, icc_profile, exif = \ - _webp.WebPDecode(self.fp.read()) + data, width, height, self.mode, icc_profile, exif = _webp.WebPDecode( + self.fp.read() + ) if icc_profile: self.info["icc_profile"] = icc_profile if exif: @@ -61,18 +60,18 @@ class WebPImageFile(ImageFile.ImageFile): self._decoder = _webp.WebPAnimDecoder(self.fp.read()) # Get info from decoder - width, height, loop_count, bgcolor, frame_count, mode = \ - self._decoder.get_info() + width, height, loop_count, bgcolor, frame_count, mode = self._decoder.get_info() self._size = width, height self.info["loop"] = loop_count - bg_a, bg_r, bg_g, bg_b = \ - (bgcolor >> 24) & 0xFF, \ - (bgcolor >> 16) & 0xFF, \ - (bgcolor >> 8) & 0xFF, \ - bgcolor & 0xFF + bg_a, bg_r, bg_g, bg_b = ( + (bgcolor >> 24) & 0xFF, + (bgcolor >> 16) & 0xFF, + (bgcolor >> 8) & 0xFF, + bgcolor & 0xFF, + ) self.info["background"] = (bg_r, bg_g, bg_b, bg_a) self._n_frames = frame_count - self.mode = 'RGB' if mode == 'RGBX' else mode + self.mode = "RGB" if mode == "RGBX" else mode self.rawmode = mode self.tile = [] @@ -92,8 +91,9 @@ class WebPImageFile(ImageFile.ImageFile): self.seek(0) def _getexif(self): - from .JpegImagePlugin import _getexif - return _getexif(self) + if "exif" not in self.info: + return None + return dict(self.getexif()) @property def n_frames(self): @@ -130,7 +130,7 @@ class WebPImageFile(ImageFile.ImageFile): # Check if an error occurred if ret is None: - self._reset() # Reset just to be safe + self._reset() # Reset just to be safe self.seek(0) raise EOFError("failed to decode next frame in WebP file") @@ -145,11 +145,11 @@ class WebPImageFile(ImageFile.ImageFile): def _seek(self, frame): if self.__physical_frame == frame: - return # Nothing to do + return # Nothing to do if frame < self.__physical_frame: - self._reset() # Rewind to beginning + self._reset() # Rewind to beginning while self.__physical_frame < frame: - self._get_next() # Advance to the requested frame + self._get_next() # Advance to the requested frame def load(self): if _webp.HAVE_WEBPANIM: @@ -163,7 +163,7 @@ class WebPImageFile(ImageFile.ImageFile): self.__loaded = self.__logical_frame # Set tile - if self.fp: + if self.fp and self._exclusive_fp: self.fp.close() self.fp = BytesIO(data) self.tile = [("raw", (0, 0) + self.size, 0, self.rawmode)] @@ -184,13 +184,25 @@ def _save_all(im, fp, filename): # If total frame count is 1, then save using the legacy API, which # will preserve non-alpha modes total = 0 - for ims in [im]+append_images: - total += 1 if not hasattr(ims, "n_frames") else ims.n_frames + for ims in [im] + append_images: + total += getattr(ims, "n_frames", 1) if total == 1: _save(im, fp, filename) return - background = encoderinfo.get("background", (0, 0, 0, 0)) + background = (0, 0, 0, 0) + if "background" in encoderinfo: + background = encoderinfo["background"] + elif "background" in im.info: + background = im.info["background"] + if isinstance(background, int): + # GifImagePlugin stores a global color table index in + # info["background"]. So it must be converted to an RGBA value + palette = im.getpalette() + if palette: + r, g, b = palette[background * 3 : (background + 1) * 3] + background = (r, g, b, 0) + duration = im.encoderinfo.get("duration", 0) loop = im.encoderinfo.get("loop", 0) minimize_size = im.encoderinfo.get("minimize_size", False) @@ -203,6 +215,8 @@ def _save_all(im, fp, filename): method = im.encoderinfo.get("method", 0) icc_profile = im.encoderinfo.get("icc_profile", "") exif = im.encoderinfo.get("exif", "") + if isinstance(exif, Image.Exif): + exif = exif.tobytes() xmp = im.encoderinfo.get("xmp", "") if allow_mixed: lossless = False @@ -214,10 +228,15 @@ def _save_all(im, fp, filename): kmax = 17 if lossless else 5 # Validate background color - if (not isinstance(background, (list, tuple)) or len(background) != 4 or - not all(v >= 0 and v < 256 for v in background)): - raise IOError("Background color is not an RGBA tuple clamped " - "to (0-255): %s" % str(background)) + if ( + not isinstance(background, (list, tuple)) + or len(background) != 4 + or not all(v >= 0 and v < 256 for v in background) + ): + raise IOError( + "Background color is not an RGBA tuple clamped to (0-255): %s" + % str(background) + ) # Convert to packed uint bg_r, bg_g, bg_b, bg_a = background @@ -225,13 +244,15 @@ def _save_all(im, fp, filename): # Setup the WebP animation encoder enc = _webp.WebPAnimEncoder( - im.size[0], im.size[1], + im.size[0], + im.size[1], background, loop, minimize_size, - kmin, kmax, + kmin, + kmax, allow_mixed, - verbose + verbose, ) # Add each frame @@ -239,12 +260,9 @@ def _save_all(im, fp, filename): timestamp = 0 cur_idx = im.tell() try: - for ims in [im]+append_images: + for ims in [im] + append_images: # Get # of frames in this image - if not hasattr(ims, "n_frames"): - nfr = 1 - else: - nfr = ims.n_frames + nfr = getattr(ims, "n_frames", 1) for idx in range(nfr): ims.seek(idx) @@ -254,24 +272,28 @@ def _save_all(im, fp, filename): frame = ims rawmode = ims.mode if ims.mode not in _VALID_WEBP_MODES: - alpha = 'A' in ims.mode or 'a' in ims.mode \ - or (ims.mode == 'P' and 'A' in ims.im.getpalettemode()) - rawmode = 'RGBA' if alpha else 'RGB' + alpha = ( + "A" in ims.mode + or "a" in ims.mode + or (ims.mode == "P" and "A" in ims.im.getpalettemode()) + ) + rawmode = "RGBA" if alpha else "RGB" frame = ims.convert(rawmode) - if rawmode == 'RGB': + if rawmode == "RGB": # For faster conversion, use RGBX - rawmode = 'RGBX' + rawmode = "RGBX" # Append the frame to the animation encoder enc.add( - frame.tobytes('raw', rawmode), + frame.tobytes("raw", rawmode), timestamp, - frame.size[0], frame.size[1], + frame.size[0], + frame.size[1], rawmode, lossless, quality, - method + method, ) # Update timestamp and frame index @@ -285,11 +307,7 @@ def _save_all(im, fp, filename): im.seek(cur_idx) # Force encoder to flush frames - enc.add( - None, - timestamp, - 0, 0, "", lossless, quality, 0 - ) + enc.add(None, timestamp, 0, 0, "", lossless, quality, 0) # Get the final output from the encoder data = enc.assemble(icc_profile, exif, xmp) @@ -304,12 +322,17 @@ def _save(im, fp, filename): quality = im.encoderinfo.get("quality", 80) icc_profile = im.encoderinfo.get("icc_profile", "") exif = im.encoderinfo.get("exif", "") + if isinstance(exif, Image.Exif): + exif = exif.tobytes() xmp = im.encoderinfo.get("xmp", "") if im.mode not in _VALID_WEBP_LEGACY_MODES: - alpha = 'A' in im.mode or 'a' in im.mode \ - or (im.mode == 'P' and 'A' in im.im.getpalettemode()) - im = im.convert('RGBA' if alpha else 'RGB') + alpha = ( + "A" in im.mode + or "a" in im.mode + or (im.mode == "P" and "A" in im.im.getpalettemode()) + ) + im = im.convert("RGBA" if alpha else "RGB") data = _webp.WebPEncode( im.tobytes(), @@ -320,7 +343,7 @@ def _save(im, fp, filename): im.mode, icc_profile, exif, - xmp + xmp, ) if data is None: raise IOError("cannot write file as WebP (encoder returned None)") diff --git a/server/www/packages/packages-linux/x64/PIL/WmfImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/WmfImagePlugin.py index 81699bd..416af6f 100644 --- a/server/www/packages/packages-linux/x64/PIL/WmfImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/WmfImagePlugin.py @@ -22,11 +22,11 @@ from __future__ import print_function from . import Image, ImageFile -from ._binary import i16le as word, si16le as short, \ - i32le as dword, si32le as _long +from ._binary import i16le as word, i32le as dword, si16le as short, si32le as _long from ._util import py3 - +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.2" _handler = None @@ -49,7 +49,6 @@ if hasattr(Image.core, "drawwmf"): # install default handler (windows only) class WmfHandler(object): - def open(self, im): im.mode = "RGB" self.bbox = im.info["wmf_bbox"] @@ -57,10 +56,14 @@ if hasattr(Image.core, "drawwmf"): def load(self, im): im.fp.seek(0) # rewind return Image.frombytes( - "RGB", im.size, + "RGB", + im.size, Image.core.drawwmf(im.fp.read(), im.size, self.bbox), - "raw", "BGR", (im.size[0]*3 + 3) & -4, -1 - ) + "raw", + "BGR", + (im.size[0] * 3 + 3) & -4, + -1, + ) register_handler(WmfHandler()) @@ -71,14 +74,14 @@ if hasattr(Image.core, "drawwmf"): def _accept(prefix): return ( - prefix[:6] == b"\xd7\xcd\xc6\x9a\x00\x00" or - prefix[:4] == b"\x01\x00\x00\x00" - ) + prefix[:6] == b"\xd7\xcd\xc6\x9a\x00\x00" or prefix[:4] == b"\x01\x00\x00\x00" + ) ## # Image plugin for Windows metafiles. + class WmfStubImageFile(ImageFile.StubImageFile): format = "WMF" @@ -129,8 +132,8 @@ class WmfStubImageFile(ImageFile.StubImageFile): size = x1 - x0, y1 - y0 # calculate dots per inch from bbox and frame - xdpi = 2540 * (x1 - y0) // (frame[2] - frame[0]) - ydpi = 2540 * (y1 - y0) // (frame[3] - frame[1]) + xdpi = int(2540.0 * (x1 - y0) / (frame[2] - frame[0]) + 0.5) + ydpi = int(2540.0 * (y1 - y0) / (frame[3] - frame[1]) + 0.5) self.info["wmf_bbox"] = x0, y0, x1, y1 @@ -158,6 +161,7 @@ def _save(im, fp, filename): raise IOError("WMF save handler not installed") _handler.save(im, fp, filename) + # # -------------------------------------------------------------------- # Registry stuff diff --git a/server/www/packages/packages-linux/x64/PIL/XVThumbImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/XVThumbImagePlugin.py index 8cdd848..aa3536d 100644 --- a/server/www/packages/packages-linux/x64/PIL/XVThumbImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/XVThumbImagePlugin.py @@ -20,6 +20,8 @@ from . import Image, ImageFile, ImagePalette from ._binary import i8, o8 +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.1" _MAGIC = b"P7 332" @@ -29,7 +31,9 @@ PALETTE = b"" for r in range(8): for g in range(8): for b in range(4): - PALETTE = PALETTE + (o8((r*255)//7)+o8((g*255)//7)+o8((b*255)//3)) + PALETTE = PALETTE + ( + o8((r * 255) // 7) + o8((g * 255) // 7) + o8((b * 255) // 3) + ) def _accept(prefix): @@ -39,6 +43,7 @@ def _accept(prefix): ## # Image plugin for XV thumbnail images. + class XVThumbImageFile(ImageFile.ImageFile): format = "XVThumb" @@ -69,10 +74,7 @@ class XVThumbImageFile(ImageFile.ImageFile): self.palette = ImagePalette.raw("RGB", PALETTE) - self.tile = [ - ("raw", (0, 0)+self.size, - self.fp.tell(), (self.mode, 0, 1) - )] + self.tile = [("raw", (0, 0) + self.size, self.fp.tell(), (self.mode, 0, 1))] # -------------------------------------------------------------------- diff --git a/server/www/packages/packages-linux/x64/PIL/XbmImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/XbmImagePlugin.py index 0cccda1..bc825c3 100644 --- a/server/www/packages/packages-linux/x64/PIL/XbmImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/XbmImagePlugin.py @@ -20,8 +20,11 @@ # import re + from . import Image, ImageFile +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.6" # XBM header @@ -43,6 +46,7 @@ def _accept(prefix): ## # Image plugin for X11 bitmaps. + class XbmImageFile(ImageFile.ImageFile): format = "XBM" @@ -58,14 +62,12 @@ class XbmImageFile(ImageFile.ImageFile): ysize = int(m.group("height")) if m.group("hotspot"): - self.info["hotspot"] = ( - int(m.group("xhot")), int(m.group("yhot")) - ) + self.info["hotspot"] = (int(m.group("xhot")), int(m.group("yhot"))) self.mode = "1" self._size = xsize, ysize - self.tile = [("xbm", (0, 0)+self.size, m.end(), None)] + self.tile = [("xbm", (0, 0) + self.size, m.end(), None)] def _save(im, fp, filename): @@ -73,17 +75,17 @@ def _save(im, fp, filename): if im.mode != "1": raise IOError("cannot write mode %s as XBM" % im.mode) - fp.write(("#define im_width %d\n" % im.size[0]).encode('ascii')) - fp.write(("#define im_height %d\n" % im.size[1]).encode('ascii')) + fp.write(("#define im_width %d\n" % im.size[0]).encode("ascii")) + fp.write(("#define im_height %d\n" % im.size[1]).encode("ascii")) hotspot = im.encoderinfo.get("hotspot") if hotspot: - fp.write(("#define im_x_hot %d\n" % hotspot[0]).encode('ascii')) - fp.write(("#define im_y_hot %d\n" % hotspot[1]).encode('ascii')) + fp.write(("#define im_x_hot %d\n" % hotspot[0]).encode("ascii")) + fp.write(("#define im_y_hot %d\n" % hotspot[1]).encode("ascii")) fp.write(b"static char im_bits[] = {\n") - ImageFile._save(im, fp, [("xbm", (0, 0)+im.size, 0, None)]) + ImageFile._save(im, fp, [("xbm", (0, 0) + im.size, 0, None)]) fp.write(b"};\n") diff --git a/server/www/packages/packages-linux/x64/PIL/XpmImagePlugin.py b/server/www/packages/packages-linux/x64/PIL/XpmImagePlugin.py index 02bc28a..2751488 100644 --- a/server/www/packages/packages-linux/x64/PIL/XpmImagePlugin.py +++ b/server/www/packages/packages-linux/x64/PIL/XpmImagePlugin.py @@ -16,13 +16,16 @@ import re + from . import Image, ImageFile, ImagePalette from ._binary import i8, o8 +# __version__ is deprecated and will be removed in a future version. Use +# PIL.__version__ instead. __version__ = "0.2" # XPM header -xpm_head = re.compile(b"\"([0-9]*) ([0-9]*) ([0-9]*) ([0-9]*)") +xpm_head = re.compile(b'"([0-9]*) ([0-9]*) ([0-9]*) ([0-9]*)') def _accept(prefix): @@ -32,6 +35,7 @@ def _accept(prefix): ## # Image plugin for X11 pixel maps. + class XpmImageFile(ImageFile.ImageFile): format = "XPM" @@ -67,9 +71,9 @@ class XpmImageFile(ImageFile.ImageFile): for i in range(pal): s = self.fp.readline() - if s[-2:] == b'\r\n': + if s[-2:] == b"\r\n": s = s[:-2] - elif s[-1:] in b'\r\n': + elif s[-1:] in b"\r\n": s = s[:-1] c = i8(s[1]) @@ -80,15 +84,15 @@ class XpmImageFile(ImageFile.ImageFile): if s[i] == b"c": # process colour key - rgb = s[i+1] + rgb = s[i + 1] if rgb == b"None": self.info["transparency"] = c elif rgb[0:1] == b"#": # FIXME: handle colour names (see ImagePalette.py) rgb = int(rgb[1:], 16) - palette[c] = (o8((rgb >> 16) & 255) + - o8((rgb >> 8) & 255) + - o8(rgb & 255)) + palette[c] = ( + o8((rgb >> 16) & 255) + o8((rgb >> 8) & 255) + o8(rgb & 255) + ) else: # unknown colour raise ValueError("cannot read this XPM file") @@ -102,7 +106,7 @@ class XpmImageFile(ImageFile.ImageFile): self.mode = "P" self.palette = ImagePalette.raw("RGB", b"".join(palette)) - self.tile = [("raw", (0, 0)+self.size, self.fp.tell(), ("P", 0, 1))] + self.tile = [("raw", (0, 0) + self.size, self.fp.tell(), ("P", 0, 1))] def load_read(self, bytes): @@ -114,10 +118,11 @@ class XpmImageFile(ImageFile.ImageFile): s = [None] * ysize for i in range(ysize): - s[i] = self.fp.readline()[1:xsize+1].ljust(xsize) + s[i] = self.fp.readline()[1 : xsize + 1].ljust(xsize) return b"".join(s) + # # Registry diff --git a/server/www/packages/packages-linux/x64/PIL/__init__.py b/server/www/packages/packages-linux/x64/PIL/__init__.py index bc8cfed..59eccc9 100644 --- a/server/www/packages/packages-linux/x64/PIL/__init__.py +++ b/server/www/packages/packages-linux/x64/PIL/__init__.py @@ -16,57 +16,58 @@ PIL.VERSION is the old PIL version and will be removed in the future. from . import _version -# VERSION is deprecated and will be removed in Pillow 6.0.0. -# PILLOW_VERSION is deprecated and will be removed after that. +# VERSION was removed in Pillow 6.0.0. +# PILLOW_VERSION is deprecated and will be removed in Pillow 7.0.0. # Use __version__ instead. -VERSION = '1.1.7' # PIL Version PILLOW_VERSION = __version__ = _version.__version__ del _version -_plugins = ['BlpImagePlugin', - 'BmpImagePlugin', - 'BufrStubImagePlugin', - 'CurImagePlugin', - 'DcxImagePlugin', - 'DdsImagePlugin', - 'EpsImagePlugin', - 'FitsStubImagePlugin', - 'FliImagePlugin', - 'FpxImagePlugin', - 'FtexImagePlugin', - 'GbrImagePlugin', - 'GifImagePlugin', - 'GribStubImagePlugin', - 'Hdf5StubImagePlugin', - 'IcnsImagePlugin', - 'IcoImagePlugin', - 'ImImagePlugin', - 'ImtImagePlugin', - 'IptcImagePlugin', - 'JpegImagePlugin', - 'Jpeg2KImagePlugin', - 'McIdasImagePlugin', - 'MicImagePlugin', - 'MpegImagePlugin', - 'MpoImagePlugin', - 'MspImagePlugin', - 'PalmImagePlugin', - 'PcdImagePlugin', - 'PcxImagePlugin', - 'PdfImagePlugin', - 'PixarImagePlugin', - 'PngImagePlugin', - 'PpmImagePlugin', - 'PsdImagePlugin', - 'SgiImagePlugin', - 'SpiderImagePlugin', - 'SunImagePlugin', - 'TgaImagePlugin', - 'TiffImagePlugin', - 'WebPImagePlugin', - 'WmfImagePlugin', - 'XbmImagePlugin', - 'XpmImagePlugin', - 'XVThumbImagePlugin'] +_plugins = [ + "BlpImagePlugin", + "BmpImagePlugin", + "BufrStubImagePlugin", + "CurImagePlugin", + "DcxImagePlugin", + "DdsImagePlugin", + "EpsImagePlugin", + "FitsStubImagePlugin", + "FliImagePlugin", + "FpxImagePlugin", + "FtexImagePlugin", + "GbrImagePlugin", + "GifImagePlugin", + "GribStubImagePlugin", + "Hdf5StubImagePlugin", + "IcnsImagePlugin", + "IcoImagePlugin", + "ImImagePlugin", + "ImtImagePlugin", + "IptcImagePlugin", + "JpegImagePlugin", + "Jpeg2KImagePlugin", + "McIdasImagePlugin", + "MicImagePlugin", + "MpegImagePlugin", + "MpoImagePlugin", + "MspImagePlugin", + "PalmImagePlugin", + "PcdImagePlugin", + "PcxImagePlugin", + "PdfImagePlugin", + "PixarImagePlugin", + "PngImagePlugin", + "PpmImagePlugin", + "PsdImagePlugin", + "SgiImagePlugin", + "SpiderImagePlugin", + "SunImagePlugin", + "TgaImagePlugin", + "TiffImagePlugin", + "WebPImagePlugin", + "WmfImagePlugin", + "XbmImagePlugin", + "XpmImagePlugin", + "XVThumbImagePlugin", +] diff --git a/server/www/packages/packages-linux/x64/PIL/__main__.py b/server/www/packages/packages-linux/x64/PIL/__main__.py new file mode 100644 index 0000000..a05323f --- /dev/null +++ b/server/www/packages/packages-linux/x64/PIL/__main__.py @@ -0,0 +1,3 @@ +from .features import pilinfo + +pilinfo() diff --git a/server/www/packages/packages-linux/x64/PIL/_binary.py b/server/www/packages/packages-linux/x64/PIL/_binary.py index 767c13b..53b1ca9 100644 --- a/server/www/packages/packages-linux/x64/PIL/_binary.py +++ b/server/www/packages/packages-linux/x64/PIL/_binary.py @@ -11,16 +11,21 @@ # See the README file for information on usage and redistribution. # -from struct import unpack_from, pack +from struct import pack, unpack_from + from ._util import py3 if py3: + def i8(c): return c if c.__class__ is int else c[0] def o8(i): return bytes((i & 255,)) + + else: + def i8(c): return ord(c) @@ -33,8 +38,8 @@ def i16le(c, o=0): """ Converts a 2-bytes (16 bits) string to an unsigned integer. - c: string containing bytes to convert - o: offset of bytes to convert in string + :param c: string containing bytes to convert + :param o: offset of bytes to convert in string """ return unpack_from(" 2: else: from Tkinter import tkinter as tk -if hasattr(sys, 'pypy_find_executable'): +if hasattr(sys, "pypy_find_executable"): # Tested with packages at https://bitbucket.org/pypy/pypy/downloads. # PyPies 1.6, 2.0 do not have tkinter built in. PyPy3-2.3.1 gives an # OSError trying to import tkinter. Otherwise: diff --git a/server/www/packages/packages-linux/x64/PIL/_util.py b/server/www/packages/packages-linux/x64/PIL/_util.py index e6989d6..59964c7 100644 --- a/server/www/packages/packages-linux/x64/PIL/_util.py +++ b/server/www/packages/packages-linux/x64/PIL/_util.py @@ -2,19 +2,32 @@ import os import sys py3 = sys.version_info.major >= 3 +py36 = sys.version_info[0:2] >= (3, 6) if py3: + def isStringType(t): return isinstance(t, str) - def isPath(f): - return isinstance(f, (bytes, str)) + if py36: + from pathlib import Path + + def isPath(f): + return isinstance(f, (bytes, str, Path)) + + else: + + def isPath(f): + return isinstance(f, (bytes, str)) + + else: + def isStringType(t): - return isinstance(t, basestring) + return isinstance(t, basestring) # noqa: F821 def isPath(f): - return isinstance(f, basestring) + return isinstance(f, basestring) # noqa: F821 # Checks if an object is a string, and that it points to a directory. diff --git a/server/www/packages/packages-linux/x64/PIL/_version.py b/server/www/packages/packages-linux/x64/PIL/_version.py index b5e4f0d..e2747ce 100644 --- a/server/www/packages/packages-linux/x64/PIL/_version.py +++ b/server/www/packages/packages-linux/x64/PIL/_version.py @@ -1,2 +1,2 @@ # Master version for Pillow -__version__ = '5.3.0' +__version__ = "6.2.1" diff --git a/server/www/packages/packages-linux/x64/PIL/_webp.cpython-37m-x86_64-linux-gnu.so b/server/www/packages/packages-linux/x64/PIL/_webp.cpython-37m-x86_64-linux-gnu.so index 176915a..24eca11 100755 Binary files a/server/www/packages/packages-linux/x64/PIL/_webp.cpython-37m-x86_64-linux-gnu.so and b/server/www/packages/packages-linux/x64/PIL/_webp.cpython-37m-x86_64-linux-gnu.so differ diff --git a/server/www/packages/packages-linux/x64/PIL/features.py b/server/www/packages/packages-linux/x64/PIL/features.py index 9926445..9fd5223 100644 --- a/server/www/packages/packages-linux/x64/PIL/features.py +++ b/server/www/packages/packages-linux/x64/PIL/features.py @@ -1,3 +1,11 @@ +from __future__ import print_function, unicode_literals + +import collections +import os +import sys + +import PIL + from . import Image modules = { @@ -26,12 +34,7 @@ def get_supported_modules(): return [f for f in modules if check_module(f)] -codecs = { - "jpg": "jpeg", - "jpg_2000": "jpeg2k", - "zlib": "zip", - "libtiff": "libtiff" -} +codecs = {"jpg": "jpeg", "jpg_2000": "jpeg2k", "zlib": "zip", "libtiff": "libtiff"} def check_codec(feature): @@ -48,10 +51,11 @@ def get_supported_codecs(): features = { - "webp_anim": ("PIL._webp", 'HAVE_WEBPANIM'), - "webp_mux": ("PIL._webp", 'HAVE_WEBPMUX'), + "webp_anim": ("PIL._webp", "HAVE_WEBPANIM"), + "webp_mux": ("PIL._webp", "HAVE_WEBPMUX"), "transp_webp": ("PIL._webp", "HAVE_TRANSPARENCY"), - "raqm": ("PIL._imagingft", "HAVE_RAQM") + "raqm": ("PIL._imagingft", "HAVE_RAQM"), + "libjpeg_turbo": ("PIL._imaging", "HAVE_LIBJPEGTURBO"), } @@ -62,7 +66,7 @@ def check_feature(feature): module, flag = features[feature] try: - imported_module = __import__(module, fromlist=['PIL']) + imported_module = __import__(module, fromlist=["PIL"]) return getattr(imported_module, flag) except ImportError: return None @@ -73,9 +77,14 @@ def get_supported_features(): def check(feature): - return (feature in modules and check_module(feature) or - feature in codecs and check_codec(feature) or - feature in features and check_feature(feature)) + return ( + feature in modules + and check_module(feature) + or feature in codecs + and check_codec(feature) + or feature in features + and check_feature(feature) + ) def get_supported(): @@ -83,3 +92,78 @@ def get_supported(): ret.extend(get_supported_features()) ret.extend(get_supported_codecs()) return ret + + +def pilinfo(out=None): + if out is None: + out = sys.stdout + + Image.init() + + print("-" * 68, file=out) + print("Pillow {}".format(PIL.__version__), file=out) + print("-" * 68, file=out) + print( + "Python modules loaded from {}".format(os.path.dirname(Image.__file__)), + file=out, + ) + print( + "Binary modules loaded from {}".format(os.path.dirname(Image.core.__file__)), + file=out, + ) + print("-" * 68, file=out) + + v = sys.version.splitlines() + print("Python {}".format(v[0].strip()), file=out) + for v in v[1:]: + print(" {}".format(v.strip()), file=out) + print("-" * 68, file=out) + + for name, feature in [ + ("pil", "PIL CORE"), + ("tkinter", "TKINTER"), + ("freetype2", "FREETYPE2"), + ("littlecms2", "LITTLECMS2"), + ("webp", "WEBP"), + ("transp_webp", "WEBP Transparency"), + ("webp_mux", "WEBPMUX"), + ("webp_anim", "WEBP Animation"), + ("jpg", "JPEG"), + ("jpg_2000", "OPENJPEG (JPEG2000)"), + ("zlib", "ZLIB (PNG/ZIP)"), + ("libtiff", "LIBTIFF"), + ("raqm", "RAQM (Bidirectional Text)"), + ]: + if check(name): + print("---", feature, "support ok", file=out) + else: + print("***", feature, "support not installed", file=out) + print("-" * 68, file=out) + + extensions = collections.defaultdict(list) + for ext, i in Image.EXTENSION.items(): + extensions[i].append(ext) + + for i in sorted(Image.ID): + line = "{}".format(i) + if i in Image.MIME: + line = "{} {}".format(line, Image.MIME[i]) + print(line, file=out) + + if i in extensions: + print("Extensions: {}".format(", ".join(sorted(extensions[i]))), file=out) + + features = [] + if i in Image.OPEN: + features.append("open") + if i in Image.SAVE: + features.append("save") + if i in Image.SAVE_ALL: + features.append("save_all") + if i in Image.DECODERS: + features.append("decode") + if i in Image.ENCODERS: + features.append("encode") + + print("Features: {}".format(", ".join(features)), file=out) + print("-" * 68, file=out) diff --git a/server/www/packages/packages-linux/x64/cryptography/__about__.py b/server/www/packages/packages-linux/x64/cryptography/__about__.py index ac18bb4..4055e29 100644 --- a/server/www/packages/packages-linux/x64/cryptography/__about__.py +++ b/server/www/packages/packages-linux/x64/cryptography/__about__.py @@ -14,10 +14,10 @@ __summary__ = ("cryptography is a package which provides cryptographic recipes" " and primitives to Python developers.") __uri__ = "https://github.com/pyca/cryptography" -__version__ = "2.4.2" +__version__ = "2.8" __author__ = "The cryptography developers" __email__ = "cryptography-dev@python.org" __license__ = "BSD or Apache License, Version 2.0" -__copyright__ = "Copyright 2013-2017 {0}".format(__author__) +__copyright__ = "Copyright 2013-2019 {}".format(__author__) diff --git a/server/www/packages/packages-linux/x64/cryptography/exceptions.py b/server/www/packages/packages-linux/x64/cryptography/exceptions.py index 648cf9d..1d52d7d 100644 --- a/server/www/packages/packages-linux/x64/cryptography/exceptions.py +++ b/server/www/packages/packages-linux/x64/cryptography/exceptions.py @@ -19,6 +19,7 @@ class _Reasons(Enum): UNSUPPORTED_X509 = 8 UNSUPPORTED_EXCHANGE_ALGORITHM = 9 UNSUPPORTED_DIFFIE_HELLMAN = 10 + UNSUPPORTED_MAC = 11 class UnsupportedAlgorithm(Exception): diff --git a/server/www/packages/packages-linux/x64/cryptography/fernet.py b/server/www/packages/packages-linux/x64/cryptography/fernet.py index ac2dd0b..b990def 100644 --- a/server/www/packages/packages-linux/x64/cryptography/fernet.py +++ b/server/www/packages/packages-linux/x64/cryptography/fernet.py @@ -12,6 +12,7 @@ import time import six +from cryptography import utils from cryptography.exceptions import InvalidSignature from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes, padding @@ -51,8 +52,7 @@ class Fernet(object): return self._encrypt_from_parts(data, current_time, iv) def _encrypt_from_parts(self, data, current_time, iv): - if not isinstance(data, bytes): - raise TypeError("data must be bytes.") + utils._check_bytes("data", data) padder = padding.PKCS7(algorithms.AES.block_size).padder() padded_data = padder.update(data) + padder.finalize() @@ -82,9 +82,7 @@ class Fernet(object): @staticmethod def _get_unverified_token_data(token): - if not isinstance(token, bytes): - raise TypeError("token must be bytes.") - + utils._check_bytes("token", token) try: data = base64.urlsafe_b64decode(token) except (TypeError, binascii.Error): diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/_der.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/_der.py new file mode 100644 index 0000000..51518d6 --- /dev/null +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/_der.py @@ -0,0 +1,156 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import six + +from cryptography.utils import int_from_bytes, int_to_bytes + + +# This module contains a lightweight DER encoder and decoder. See X.690 for the +# specification. This module intentionally does not implement the more complex +# BER encoding, only DER. +# +# Note this implementation treats an element's constructed bit as part of the +# tag. This is fine for DER, where the bit is always computable from the type. + + +CONSTRUCTED = 0x20 +CONTEXT_SPECIFIC = 0x80 + +INTEGER = 0x02 +BIT_STRING = 0x03 +OCTET_STRING = 0x04 +NULL = 0x05 +OBJECT_IDENTIFIER = 0x06 +SEQUENCE = 0x10 | CONSTRUCTED +SET = 0x11 | CONSTRUCTED +PRINTABLE_STRING = 0x13 +UTC_TIME = 0x17 +GENERALIZED_TIME = 0x18 + + +class DERReader(object): + def __init__(self, data): + self.data = memoryview(data) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, tb): + if exc_value is None: + self.check_empty() + + def is_empty(self): + return len(self.data) == 0 + + def check_empty(self): + if not self.is_empty(): + raise ValueError("Invalid DER input: trailing data") + + def read_byte(self): + if len(self.data) < 1: + raise ValueError("Invalid DER input: insufficient data") + ret = six.indexbytes(self.data, 0) + self.data = self.data[1:] + return ret + + def read_bytes(self, n): + if len(self.data) < n: + raise ValueError("Invalid DER input: insufficient data") + ret = self.data[:n] + self.data = self.data[n:] + return ret + + def read_any_element(self): + tag = self.read_byte() + # Tag numbers 31 or higher are stored in multiple bytes. No supported + # ASN.1 types use such tags, so reject these. + if tag & 0x1f == 0x1f: + raise ValueError("Invalid DER input: unexpected high tag number") + length_byte = self.read_byte() + if length_byte & 0x80 == 0: + # If the high bit is clear, the first length byte is the length. + length = length_byte + else: + # If the high bit is set, the first length byte encodes the length + # of the length. + length_byte &= 0x7f + if length_byte == 0: + raise ValueError( + "Invalid DER input: indefinite length form is not allowed " + "in DER" + ) + length = 0 + for i in range(length_byte): + length <<= 8 + length |= self.read_byte() + if length == 0: + raise ValueError( + "Invalid DER input: length was not minimally-encoded" + ) + if length < 0x80: + # If the length could have been encoded in short form, it must + # not use long form. + raise ValueError( + "Invalid DER input: length was not minimally-encoded" + ) + body = self.read_bytes(length) + return tag, DERReader(body) + + def read_element(self, expected_tag): + tag, body = self.read_any_element() + if tag != expected_tag: + raise ValueError("Invalid DER input: unexpected tag") + return body + + def read_single_element(self, expected_tag): + with self: + return self.read_element(expected_tag) + + def read_optional_element(self, expected_tag): + if len(self.data) > 0 and six.indexbytes(self.data, 0) == expected_tag: + return self.read_element(expected_tag) + return None + + def as_integer(self): + if len(self.data) == 0: + raise ValueError("Invalid DER input: empty integer contents") + first = six.indexbytes(self.data, 0) + if first & 0x80 == 0x80: + raise ValueError("Negative DER integers are not supported") + # The first 9 bits must not all be zero or all be ones. Otherwise, the + # encoding should have been one byte shorter. + if len(self.data) > 1: + second = six.indexbytes(self.data, 1) + if first == 0 and second & 0x80 == 0: + raise ValueError( + "Invalid DER input: integer not minimally-encoded" + ) + return int_from_bytes(self.data, "big") + + +def encode_der_integer(x): + if not isinstance(x, six.integer_types): + raise ValueError("Value must be an integer") + if x < 0: + raise ValueError("Negative integers are not supported") + n = x.bit_length() // 8 + 1 + return int_to_bytes(x, n) + + +def encode_der(tag, *children): + length = 0 + for child in children: + length += len(child) + chunks = [six.int2byte(tag)] + if length < 0x80: + chunks.append(six.int2byte(length)) + else: + length_bytes = int_to_bytes(length) + chunks.append(six.int2byte(0x80 | len(length_bytes))) + chunks.append(length_bytes) + chunks.extend(children) + return b"".join(chunks) diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/_oid.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/_oid.py index cfe906c..4b08722 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/_oid.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/_oid.py @@ -50,7 +50,7 @@ class ObjectIdentifier(object): return not self == other def __repr__(self): - return "".format( + return "".format( self.dotted_string, self._name ) diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/interfaces.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/interfaces.py index 0a476b9..20f4164 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/interfaces.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/interfaces.py @@ -57,7 +57,7 @@ class HMACBackend(object): @abc.abstractmethod def create_hmac_ctx(self, key, algorithm): """ - Create a MACContext for calculating a message authentication code. + Create a context for calculating a message authentication code. """ @@ -72,7 +72,7 @@ class CMACBackend(object): @abc.abstractmethod def create_cmac_ctx(self, algorithm): """ - Create a MACContext for calculating a message authentication code. + Create a context for calculating a message authentication code. """ diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/aead.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/aead.py index 9cec3e2..0cad15c 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/aead.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/aead.py @@ -18,10 +18,10 @@ def _aead_cipher_name(cipher): if isinstance(cipher, ChaCha20Poly1305): return b"chacha20-poly1305" elif isinstance(cipher, AESCCM): - return "aes-{0}-ccm".format(len(cipher._key) * 8).encode("ascii") + return "aes-{}-ccm".format(len(cipher._key) * 8).encode("ascii") else: assert isinstance(cipher, AESGCM) - return "aes-{0}-gcm".format(len(cipher._key) * 8).encode("ascii") + return "aes-{}-gcm".format(len(cipher._key) * 8).encode("ascii") def _aead_setup(backend, cipher_name, key, nonce, tag, tag_len, operation): @@ -49,17 +49,20 @@ def _aead_setup(backend, cipher_name, key, nonce, tag, tag_len, operation): ctx, backend._lib.EVP_CTRL_AEAD_SET_TAG, len(tag), tag ) backend.openssl_assert(res != 0) - else: + elif cipher_name.endswith(b"-ccm"): res = backend._lib.EVP_CIPHER_CTX_ctrl( ctx, backend._lib.EVP_CTRL_AEAD_SET_TAG, tag_len, backend._ffi.NULL ) + backend.openssl_assert(res != 0) + nonce_ptr = backend._ffi.from_buffer(nonce) + key_ptr = backend._ffi.from_buffer(key) res = backend._lib.EVP_CipherInit_ex( ctx, backend._ffi.NULL, backend._ffi.NULL, - key, - nonce, + key_ptr, + nonce_ptr, int(operation == _ENCRYPT) ) backend.openssl_assert(res != 0) diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/backend.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/backend.py index 19734a5..7e9fa20 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/backend.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/backend.py @@ -5,18 +5,19 @@ from __future__ import absolute_import, division, print_function import base64 -import calendar import collections import contextlib import itertools from contextlib import contextmanager -import asn1crypto.core - import six +from six.moves import range from cryptography import utils, x509 from cryptography.exceptions import UnsupportedAlgorithm, _Reasons +from cryptography.hazmat._der import ( + INTEGER, NULL, SEQUENCE, encode_der, encode_der_integer +) from cryptography.hazmat.backends.interfaces import ( CMACBackend, CipherBackend, DERSerializationBackend, DHBackend, DSABackend, EllipticCurveBackend, HMACBackend, HashBackend, PBKDF2HMACBackend, @@ -26,7 +27,7 @@ from cryptography.hazmat.backends.openssl import aead from cryptography.hazmat.backends.openssl.ciphers import _CipherContext from cryptography.hazmat.backends.openssl.cmac import _CMACContext from cryptography.hazmat.backends.openssl.decode_asn1 import ( - _CRL_ENTRY_REASON_ENUM_TO_CODE, _Integers + _CRL_ENTRY_REASON_ENUM_TO_CODE ) from cryptography.hazmat.backends.openssl.dh import ( _DHParameters, _DHPrivateKey, _DHPublicKey, _dh_params_dup @@ -37,6 +38,12 @@ from cryptography.hazmat.backends.openssl.dsa import ( from cryptography.hazmat.backends.openssl.ec import ( _EllipticCurvePrivateKey, _EllipticCurvePublicKey ) +from cryptography.hazmat.backends.openssl.ed25519 import ( + _Ed25519PrivateKey, _Ed25519PublicKey +) +from cryptography.hazmat.backends.openssl.ed448 import ( + _ED448_KEY_SIZE, _Ed448PrivateKey, _Ed448PublicKey +) from cryptography.hazmat.backends.openssl.encode_asn1 import ( _CRL_ENTRY_EXTENSION_ENCODE_HANDLERS, _CRL_EXTENSION_ENCODE_HANDLERS, _EXTENSION_ENCODE_HANDLERS, @@ -49,19 +56,27 @@ from cryptography.hazmat.backends.openssl.hmac import _HMACContext from cryptography.hazmat.backends.openssl.ocsp import ( _OCSPRequest, _OCSPResponse ) +from cryptography.hazmat.backends.openssl.poly1305 import ( + _POLY1305_KEY_SIZE, _Poly1305Context +) from cryptography.hazmat.backends.openssl.rsa import ( _RSAPrivateKey, _RSAPublicKey ) from cryptography.hazmat.backends.openssl.x25519 import ( _X25519PrivateKey, _X25519PublicKey ) +from cryptography.hazmat.backends.openssl.x448 import ( + _X448PrivateKey, _X448PublicKey +) from cryptography.hazmat.backends.openssl.x509 import ( _Certificate, _CertificateRevocationList, _CertificateSigningRequest, _RevokedCertificate ) from cryptography.hazmat.bindings.openssl import binding from cryptography.hazmat.primitives import hashes, serialization -from cryptography.hazmat.primitives.asymmetric import dsa, ec, rsa +from cryptography.hazmat.primitives.asymmetric import ( + dsa, ec, ed25519, ed448, rsa +) from cryptography.hazmat.primitives.asymmetric.padding import ( MGF1, OAEP, PKCS1v15, PSS ) @@ -72,6 +87,7 @@ from cryptography.hazmat.primitives.ciphers.modes import ( CBC, CFB, CFB8, CTR, ECB, GCM, OFB, XTS ) from cryptography.hazmat.primitives.kdf import scrypt +from cryptography.hazmat.primitives.serialization import ssh from cryptography.x509 import ocsp @@ -115,21 +131,23 @@ class Backend(object): return binding._openssl_assert(self._lib, ok) def activate_builtin_random(self): - # Obtain a new structural reference. - e = self._lib.ENGINE_get_default_RAND() - if e != self._ffi.NULL: - self._lib.ENGINE_unregister_RAND(e) - # Reset the RNG to use the new engine. - self._lib.RAND_cleanup() - # decrement the structural reference from get_default_RAND - res = self._lib.ENGINE_finish(e) - self.openssl_assert(res == 1) + if self._lib.Cryptography_HAS_ENGINE: + # Obtain a new structural reference. + e = self._lib.ENGINE_get_default_RAND() + if e != self._ffi.NULL: + self._lib.ENGINE_unregister_RAND(e) + # Reset the RNG to use the built-in. + res = self._lib.RAND_set_rand_method(self._ffi.NULL) + self.openssl_assert(res == 1) + # decrement the structural reference from get_default_RAND + res = self._lib.ENGINE_finish(e) + self.openssl_assert(res == 1) @contextlib.contextmanager def _get_osurandom_engine(self): # Fetches an engine by id and returns it. This creates a structural # reference. - e = self._lib.ENGINE_by_id(self._binding._osrandom_engine_id) + e = self._lib.ENGINE_by_id(self._lib.Cryptography_osrandom_engine_id) self.openssl_assert(e != self._ffi.NULL) # Initialize the engine for use. This adds a functional reference. res = self._lib.ENGINE_init(e) @@ -146,14 +164,16 @@ class Backend(object): self.openssl_assert(res == 1) def activate_osrandom_engine(self): - # Unregister and free the current engine. - self.activate_builtin_random() - with self._get_osurandom_engine() as e: - # Set the engine as the default RAND provider. - res = self._lib.ENGINE_set_default_RAND(e) + if self._lib.Cryptography_HAS_ENGINE: + # Unregister and free the current engine. + self.activate_builtin_random() + with self._get_osurandom_engine() as e: + # Set the engine as the default RAND provider. + res = self._lib.ENGINE_set_default_RAND(e) + self.openssl_assert(res == 1) + # Reset the RNG to use the engine + res = self._lib.RAND_set_rand_method(self._ffi.NULL) self.openssl_assert(res == 1) - # Reset the RNG to use the new engine. - self._lib.RAND_cleanup() def osrandom_engine_implementation(self): buf = self._ffi.new("char[]", 64) @@ -183,7 +203,7 @@ class Backend(object): def _evp_md_from_algorithm(self, algorithm): if algorithm.name == "blake2b" or algorithm.name == "blake2s": - alg = "{0}{1}".format( + alg = "{}{}".format( algorithm.name, algorithm.digest_size * 8 ).encode("ascii") else: @@ -217,7 +237,7 @@ class Backend(object): def register_cipher_adapter(self, cipher_cls, mode_cls, adapter): if (cipher_cls, mode_cls) in self._cipher_registry: - raise ValueError("Duplicate registration for: {0} {1}.".format( + raise ValueError("Duplicate registration for: {} {}.".format( cipher_cls, mode_cls) ) self._cipher_registry[cipher_cls, mode_cls] = adapter @@ -292,8 +312,9 @@ class Backend(object): key_material): buf = self._ffi.new("unsigned char[]", length) evp_md = self._evp_md_non_null_from_algorithm(algorithm) + key_material_ptr = self._ffi.from_buffer(key_material) res = self._lib.PKCS5_PBKDF2_HMAC( - key_material, + key_material_ptr, len(key_material), salt, len(salt), @@ -318,7 +339,10 @@ class Backend(object): bin_len = self._lib.BN_bn2bin(bn, bin_ptr) # A zero length means the BN has value 0 self.openssl_assert(bin_len >= 0) - return int.from_bytes(self._ffi.buffer(bin_ptr)[:bin_len], "big") + val = int.from_bytes(self._ffi.buffer(bin_ptr)[:bin_len], "big") + if self._lib.BN_is_negative(bn): + val = -val + return val else: # Under Python 2 the best we can do is hex() hex_cdata = self._lib.BN_bn2hex(bn) @@ -446,13 +470,13 @@ class Backend(object): The char* is the storage for the BIO and it must stay alive until the BIO is finished with. """ - data_char_p = self._ffi.new("char[]", data) + data_ptr = self._ffi.from_buffer(data) bio = self._lib.BIO_new_mem_buf( - data_char_p, len(data) + data_ptr, len(data) ) self.openssl_assert(bio != self._ffi.NULL) - return _MemoryBIO(self._ffi.gc(bio, self._lib.BIO_free), data_char_p) + return _MemoryBIO(self._ffi.gc(bio, self._lib.BIO_free), data_ptr) def _create_mem_bio_gc(self): """ @@ -504,6 +528,18 @@ class Backend(object): self.openssl_assert(dh_cdata != self._ffi.NULL) dh_cdata = self._ffi.gc(dh_cdata, self._lib.DH_free) return _DHPrivateKey(self, dh_cdata, evp_pkey) + elif key_type == getattr(self._lib, "EVP_PKEY_ED25519", None): + # EVP_PKEY_ED25519 is not present in OpenSSL < 1.1.1 + return _Ed25519PrivateKey(self, evp_pkey) + elif key_type == getattr(self._lib, "EVP_PKEY_X448", None): + # EVP_PKEY_X448 is not present in OpenSSL < 1.1.1 + return _X448PrivateKey(self, evp_pkey) + elif key_type == getattr(self._lib, "EVP_PKEY_X25519", None): + # EVP_PKEY_X25519 is not present in OpenSSL < 1.1.0 + return _X25519PrivateKey(self, evp_pkey) + elif key_type == getattr(self._lib, "EVP_PKEY_ED448", None): + # EVP_PKEY_ED448 is not present in OpenSSL < 1.1.1 + return _Ed448PrivateKey(self, evp_pkey) else: raise UnsupportedAlgorithm("Unsupported key type.") @@ -535,6 +571,18 @@ class Backend(object): self.openssl_assert(dh_cdata != self._ffi.NULL) dh_cdata = self._ffi.gc(dh_cdata, self._lib.DH_free) return _DHPublicKey(self, dh_cdata, evp_pkey) + elif key_type == getattr(self._lib, "EVP_PKEY_ED25519", None): + # EVP_PKEY_ED25519 is not present in OpenSSL < 1.1.1 + return _Ed25519PublicKey(self, evp_pkey) + elif key_type == getattr(self._lib, "EVP_PKEY_X448", None): + # EVP_PKEY_X448 is not present in OpenSSL < 1.1.1 + return _X448PublicKey(self, evp_pkey) + elif key_type == getattr(self._lib, "EVP_PKEY_X25519", None): + # EVP_PKEY_X25519 is not present in OpenSSL < 1.1.0 + return _X25519PublicKey(self, evp_pkey) + elif key_type == getattr(self._lib, "EVP_PKEY_ED448", None): + # EVP_PKEY_X25519 is not present in OpenSSL < 1.1.1 + return _Ed448PublicKey(self, evp_pkey) else: raise UnsupportedAlgorithm("Unsupported key type.") @@ -676,10 +724,18 @@ class Backend(object): return _CMACContext(self, algorithm) def create_x509_csr(self, builder, private_key, algorithm): - if not isinstance(algorithm, hashes.HashAlgorithm): - raise TypeError('Algorithm must be a registered hash algorithm.') + if not isinstance(builder, x509.CertificateSigningRequestBuilder): + raise TypeError('Builder type mismatch.') - if ( + if isinstance(private_key, + (ed25519.Ed25519PrivateKey, ed448.Ed448PrivateKey)): + if algorithm is not None: + raise ValueError( + "algorithm must be None when signing via ed25519 or ed448" + ) + elif not isinstance(algorithm, hashes.HashAlgorithm): + raise TypeError('Algorithm must be a registered hash algorithm.') + elif ( isinstance(algorithm, hashes.MD5) and not isinstance(private_key, rsa.RSAPrivateKey) ): @@ -688,7 +744,7 @@ class Backend(object): ) # Resolve the signature algorithm. - evp_md = self._evp_md_non_null_from_algorithm(algorithm) + evp_md = self._evp_md_x509_null_if_eddsa(private_key, algorithm) # Create an empty request. x509_req = self._lib.X509_REQ_new() @@ -755,7 +811,13 @@ class Backend(object): def create_x509_certificate(self, builder, private_key, algorithm): if not isinstance(builder, x509.CertificateBuilder): raise TypeError('Builder type mismatch.') - if not isinstance(algorithm, hashes.HashAlgorithm): + if isinstance(private_key, + (ed25519.Ed25519PrivateKey, ed448.Ed448PrivateKey)): + if algorithm is not None: + raise ValueError( + "algorithm must be None when signing via ed25519 or ed448" + ) + elif not isinstance(algorithm, hashes.HashAlgorithm): raise TypeError('Algorithm must be a registered hash algorithm.') if ( @@ -763,11 +825,11 @@ class Backend(object): isinstance(private_key, rsa.RSAPrivateKey) ): raise ValueError( - "MD5 is not a supported hash algorithm for EC/DSA certificates" + "MD5 is only (reluctantly) supported for RSA certificates" ) # Resolve the signature algorithm. - evp_md = self._evp_md_non_null_from_algorithm(algorithm) + evp_md = self._evp_md_x509_null_if_eddsa(private_key, algorithm) # Create an empty certificate. x509_cert = self._lib.X509_new() @@ -796,12 +858,12 @@ class Backend(object): # Set the "not before" time. self._set_asn1_time( - self._lib.X509_get_notBefore(x509_cert), builder._not_valid_before + self._lib.X509_getm_notBefore(x509_cert), builder._not_valid_before ) # Set the "not after" time. self._set_asn1_time( - self._lib.X509_get_notAfter(x509_cert), builder._not_valid_after + self._lib.X509_getm_notAfter(x509_cert), builder._not_valid_after ) # Add extensions. @@ -835,21 +897,21 @@ class Backend(object): return _Certificate(self, x509_cert) + def _evp_md_x509_null_if_eddsa(self, private_key, algorithm): + if isinstance(private_key, + (ed25519.Ed25519PrivateKey, ed448.Ed448PrivateKey)): + # OpenSSL requires us to pass NULL for EVP_MD for ed25519/ed448 + return self._ffi.NULL + else: + return self._evp_md_non_null_from_algorithm(algorithm) + def _set_asn1_time(self, asn1_time, time): - timestamp = calendar.timegm(time.timetuple()) - res = self._lib.ASN1_TIME_set(asn1_time, timestamp) - if res == self._ffi.NULL: - errors = self._consume_errors() - self.openssl_assert( - errors[0]._lib_reason_match( - self._lib.ERR_LIB_ASN1, - self._lib.ASN1_R_ERROR_GETTING_TIME - ) - ) - raise ValueError( - "Invalid time. This error can occur if you set a time too far " - "in the future on Windows." - ) + if time.year >= 2050: + asn1_str = time.strftime('%Y%m%d%H%M%SZ').encode('ascii') + else: + asn1_str = time.strftime('%y%m%d%H%M%SZ').encode('ascii') + res = self._lib.ASN1_TIME_set_string(asn1_time, asn1_str) + self.openssl_assert(res == 1) def _create_asn1_time(self, time): asn1_time = self._lib.ASN1_TIME_new() @@ -861,7 +923,13 @@ class Backend(object): def create_x509_crl(self, builder, private_key, algorithm): if not isinstance(builder, x509.CertificateRevocationListBuilder): raise TypeError('Builder type mismatch.') - if not isinstance(algorithm, hashes.HashAlgorithm): + if isinstance(private_key, + (ed25519.Ed25519PrivateKey, ed448.Ed448PrivateKey)): + if algorithm is not None: + raise ValueError( + "algorithm must be None when signing via ed25519 or ed448" + ) + elif not isinstance(algorithm, hashes.HashAlgorithm): raise TypeError('Algorithm must be a registered hash algorithm.') if ( @@ -872,7 +940,7 @@ class Backend(object): "MD5 is not a supported hash algorithm for EC/DSA CRLs" ) - evp_md = self._evp_md_non_null_from_algorithm(algorithm) + evp_md = self._evp_md_x509_null_if_eddsa(private_key, algorithm) # Create an empty CRL. x509_crl = self._lib.X509_CRL_new() @@ -959,19 +1027,24 @@ class Backend(object): value = _encode_asn1_str_gc(self, extension.value.value) return self._create_raw_x509_extension(extension, value) elif isinstance(extension.value, x509.TLSFeature): - asn1 = _Integers([x.value for x in extension.value]).dump() + asn1 = encode_der( + SEQUENCE, + *[ + encode_der(INTEGER, encode_der_integer(x.value)) + for x in extension.value + ] + ) value = _encode_asn1_str_gc(self, asn1) return self._create_raw_x509_extension(extension, value) elif isinstance(extension.value, x509.PrecertPoison): - asn1 = asn1crypto.core.Null().dump() - value = _encode_asn1_str_gc(self, asn1) + value = _encode_asn1_str_gc(self, encode_der(NULL)) return self._create_raw_x509_extension(extension, value) else: try: encode = handlers[extension.oid] except KeyError: raise NotImplementedError( - 'Extension not supported: {0}'.format(extension.oid) + 'Extension not supported: {}'.format(extension.oid) ) ext_struct = encode(self, extension.value) @@ -1137,7 +1210,10 @@ class Backend(object): ) if x509 == self._ffi.NULL: self._consume_errors() - raise ValueError("Unable to load certificate") + raise ValueError( + "Unable to load certificate. See https://cryptography.io/en/la" + "test/faq/#why-can-t-i-import-my-pem-file for more details." + ) x509 = self._ffi.gc(x509, self._lib.X509_free) return _Certificate(self, x509) @@ -1159,7 +1235,10 @@ class Backend(object): ) if x509_crl == self._ffi.NULL: self._consume_errors() - raise ValueError("Unable to load CRL") + raise ValueError( + "Unable to load CRL. See https://cryptography.io/en/la" + "test/faq/#why-can-t-i-import-my-pem-file for more details." + ) x509_crl = self._ffi.gc(x509_crl, self._lib.X509_CRL_free) return _CertificateRevocationList(self, x509_crl) @@ -1181,7 +1260,10 @@ class Backend(object): ) if x509_req == self._ffi.NULL: self._consume_errors() - raise ValueError("Unable to load request") + raise ValueError( + "Unable to load request. See https://cryptography.io/en/la" + "test/faq/#why-can-t-i-import-my-pem-file for more details." + ) x509_req = self._ffi.gc(x509_req, self._lib.X509_REQ_free) return _CertificateSigningRequest(self, x509_req) @@ -1199,13 +1281,11 @@ class Backend(object): def _load_key(self, openssl_read_func, convert_func, data, password): mem_bio = self._bytes_to_bio(data) - if password is not None and not isinstance(password, bytes): - raise TypeError("Password must be bytes") - userdata = self._ffi.new("CRYPTOGRAPHY_PASSWORD_DATA *") if password is not None: - password_buf = self._ffi.new("char []", password) - userdata.password = password_buf + utils._check_byteslike("password", password) + password_ptr = self._ffi.from_buffer(password) + userdata.password = password_ptr userdata.length = len(password) evp_pkey = openssl_read_func( @@ -1228,7 +1308,7 @@ class Backend(object): else: assert userdata.error == -2 raise ValueError( - "Passwords longer than {0} bytes are not supported " + "Passwords longer than {} bytes are not supported " "by this backend.".format(userdata.maxsize - 1) ) else: @@ -1330,11 +1410,7 @@ class Backend(object): """ if self.elliptic_curve_supported(curve): - curve_nid = self._elliptic_curve_to_nid(curve) - - ec_cdata = self._lib.EC_KEY_new_by_curve_name(curve_nid) - self.openssl_assert(ec_cdata != self._ffi.NULL) - ec_cdata = self._ffi.gc(ec_cdata, self._lib.EC_KEY_free) + ec_cdata = self._ec_key_new_by_curve(curve) res = self._lib.EC_KEY_generate_key(ec_cdata) self.openssl_assert(res == 1) @@ -1344,18 +1420,14 @@ class Backend(object): return _EllipticCurvePrivateKey(self, ec_cdata, evp_pkey) else: raise UnsupportedAlgorithm( - "Backend object does not support {0}.".format(curve.name), + "Backend object does not support {}.".format(curve.name), _Reasons.UNSUPPORTED_ELLIPTIC_CURVE ) def load_elliptic_curve_private_numbers(self, numbers): public = numbers.public_numbers - curve_nid = self._elliptic_curve_to_nid(public.curve) - - ec_cdata = self._lib.EC_KEY_new_by_curve_name(curve_nid) - self.openssl_assert(ec_cdata != self._ffi.NULL) - ec_cdata = self._ffi.gc(ec_cdata, self._lib.EC_KEY_free) + ec_cdata = self._ec_key_new_by_curve(public.curve) private_value = self._ffi.gc( self._int_to_bn(numbers.private_value), self._lib.BN_clear_free @@ -1371,24 +1443,35 @@ class Backend(object): return _EllipticCurvePrivateKey(self, ec_cdata, evp_pkey) def load_elliptic_curve_public_numbers(self, numbers): - curve_nid = self._elliptic_curve_to_nid(numbers.curve) - - ec_cdata = self._lib.EC_KEY_new_by_curve_name(curve_nid) - self.openssl_assert(ec_cdata != self._ffi.NULL) - ec_cdata = self._ffi.gc(ec_cdata, self._lib.EC_KEY_free) - + ec_cdata = self._ec_key_new_by_curve(numbers.curve) ec_cdata = self._ec_key_set_public_key_affine_coordinates( ec_cdata, numbers.x, numbers.y) evp_pkey = self._ec_cdata_to_evp_pkey(ec_cdata) return _EllipticCurvePublicKey(self, ec_cdata, evp_pkey) - def derive_elliptic_curve_private_key(self, private_value, curve): - curve_nid = self._elliptic_curve_to_nid(curve) + def load_elliptic_curve_public_bytes(self, curve, point_bytes): + ec_cdata = self._ec_key_new_by_curve(curve) + group = self._lib.EC_KEY_get0_group(ec_cdata) + self.openssl_assert(group != self._ffi.NULL) + point = self._lib.EC_POINT_new(group) + self.openssl_assert(point != self._ffi.NULL) + point = self._ffi.gc(point, self._lib.EC_POINT_free) + with self._tmp_bn_ctx() as bn_ctx: + res = self._lib.EC_POINT_oct2point( + group, point, point_bytes, len(point_bytes), bn_ctx + ) + if res != 1: + self._consume_errors() + raise ValueError("Invalid public bytes for the given curve") - ec_cdata = self._lib.EC_KEY_new_by_curve_name(curve_nid) - self.openssl_assert(ec_cdata != self._ffi.NULL) - ec_cdata = self._ffi.gc(ec_cdata, self._lib.EC_KEY_free) + res = self._lib.EC_KEY_set_public_key(ec_cdata, point) + self.openssl_assert(res == 1) + evp_pkey = self._ec_cdata_to_evp_pkey(ec_cdata) + return _EllipticCurvePublicKey(self, ec_cdata, evp_pkey) + + def derive_elliptic_curve_private_key(self, private_value, curve): + ec_cdata = self._ec_key_new_by_curve(curve) get_func, group = self._ec_key_determine_group_get_func(ec_cdata) @@ -1421,6 +1504,12 @@ class Backend(object): return _EllipticCurvePrivateKey(self, ec_cdata, evp_pkey) + def _ec_key_new_by_curve(self, curve): + curve_nid = self._elliptic_curve_to_nid(curve) + ec_cdata = self._lib.EC_KEY_new_by_curve_name(curve_nid) + self.openssl_assert(ec_cdata != self._ffi.NULL) + return self._ffi.gc(ec_cdata, self._lib.EC_KEY_free) + def load_der_ocsp_request(self, data): mem_bio = self._bytes_to_bio(data) request = self._lib.d2i_OCSP_REQUEST_bio(mem_bio.bio, self._ffi.NULL) @@ -1507,7 +1596,7 @@ class Backend(object): ) self.openssl_assert(res != self._ffi.NULL) # okay, now sign the basic structure - evp_md = self._evp_md_non_null_from_algorithm(algorithm) + evp_md = self._evp_md_x509_null_if_eddsa(private_key, algorithm) responder_cert, responder_encoding = builder._responder_id flags = self._lib.OCSP_NOCERTS if responder_encoding is ocsp.OCSPResponderEncoding.HASH: @@ -1585,7 +1674,7 @@ class Backend(object): curve_nid = self._lib.OBJ_sn2nid(curve_name.encode()) if curve_nid == self._lib.NID_undef: raise UnsupportedAlgorithm( - "{0} is not a supported elliptic curve".format(curve.name), + "{} is not a supported elliptic curve".format(curve.name), _Reasons.UNSUPPORTED_ELLIPTIC_CURVE ) return curve_nid @@ -1656,6 +1745,20 @@ class Backend(object): "format must be an item from the PrivateFormat enum" ) + # X9.62 encoding is only valid for EC public keys + if encoding is serialization.Encoding.X962: + raise ValueError("X9.62 format is only valid for EC public keys") + + # Raw format and encoding are only valid for X25519, Ed25519, X448, and + # Ed448 keys. We capture those cases before this method is called so if + # we see those enum values here it means the caller has passed them to + # a key that doesn't support raw type + if format is serialization.PrivateFormat.Raw: + raise ValueError("raw format is invalid with this key or encoding") + + if encoding is serialization.Encoding.Raw: + raise ValueError("raw encoding is invalid with this key or format") + if not isinstance(encryption_algorithm, serialization.KeySerializationEncryption): raise TypeError( @@ -1715,7 +1818,7 @@ class Backend(object): write_bio = self._lib.i2d_PKCS8PrivateKey_bio key = evp_pkey else: - raise TypeError("encoding must be an item from the Encoding enum") + raise TypeError("encoding must be Encoding.PEM or Encoding.DER") bio = self._create_mem_bio_gc() res = write_bio( @@ -1748,6 +1851,23 @@ class Backend(object): if not isinstance(encoding, serialization.Encoding): raise TypeError("encoding must be an item from the Encoding enum") + # Compressed/UncompressedPoint are only valid for EC keys and those + # cases are handled by the ECPublicKey public_bytes method before this + # method is called + if format in (serialization.PublicFormat.UncompressedPoint, + serialization.PublicFormat.CompressedPoint): + raise ValueError("Point formats are not valid for this key type") + + # Raw format and encoding are only valid for X25519, Ed25519, X448, and + # Ed448 keys. We capture those cases before this method is called so if + # we see those enum values here it means the caller has passed them to + # a key that doesn't support raw type + if format is serialization.PublicFormat.Raw: + raise ValueError("raw format is invalid with this key or encoding") + + if encoding is serialization.Encoding.Raw: + raise ValueError("raw encoding is invalid with this key or format") + if ( format is serialization.PublicFormat.OpenSSH or encoding is serialization.Encoding.OpenSSH @@ -1792,22 +1912,28 @@ class Backend(object): if isinstance(key, rsa.RSAPublicKey): public_numbers = key.public_numbers() return b"ssh-rsa " + base64.b64encode( - serialization._ssh_write_string(b"ssh-rsa") + - serialization._ssh_write_mpint(public_numbers.e) + - serialization._ssh_write_mpint(public_numbers.n) + ssh._ssh_write_string(b"ssh-rsa") + + ssh._ssh_write_mpint(public_numbers.e) + + ssh._ssh_write_mpint(public_numbers.n) ) elif isinstance(key, dsa.DSAPublicKey): public_numbers = key.public_numbers() parameter_numbers = public_numbers.parameter_numbers return b"ssh-dss " + base64.b64encode( - serialization._ssh_write_string(b"ssh-dss") + - serialization._ssh_write_mpint(parameter_numbers.p) + - serialization._ssh_write_mpint(parameter_numbers.q) + - serialization._ssh_write_mpint(parameter_numbers.g) + - serialization._ssh_write_mpint(public_numbers.y) + ssh._ssh_write_string(b"ssh-dss") + + ssh._ssh_write_mpint(parameter_numbers.p) + + ssh._ssh_write_mpint(parameter_numbers.q) + + ssh._ssh_write_mpint(parameter_numbers.g) + + ssh._ssh_write_mpint(public_numbers.y) ) - else: - assert isinstance(key, ec.EllipticCurvePublicKey) + elif isinstance(key, ed25519.Ed25519PublicKey): + raw_bytes = key.public_bytes(serialization.Encoding.Raw, + serialization.PublicFormat.Raw) + return b"ssh-ed25519 " + base64.b64encode( + ssh._ssh_write_string(b"ssh-ed25519") + + ssh._ssh_write_string(raw_bytes) + ) + elif isinstance(key, ec.EllipticCurvePublicKey): public_numbers = key.public_numbers() try: curve_name = { @@ -1820,10 +1946,19 @@ class Backend(object): "Only SECP256R1, SECP384R1, and SECP521R1 curves are " "supported by the SSH public key format" ) + + point = key.public_bytes( + serialization.Encoding.X962, + serialization.PublicFormat.UncompressedPoint + ) return b"ecdsa-sha2-" + curve_name + b" " + base64.b64encode( - serialization._ssh_write_string(b"ecdsa-sha2-" + curve_name) + - serialization._ssh_write_string(curve_name) + - serialization._ssh_write_string(public_numbers.encode_point()) + ssh._ssh_write_string(b"ecdsa-sha2-" + curve_name) + + ssh._ssh_write_string(curve_name) + + ssh._ssh_write_string(point) + ) + else: + raise ValueError( + "OpenSSH encoding is not supported for this key type" ) def _parameter_bytes(self, encoding, format, cdata): @@ -2027,6 +2162,11 @@ class Backend(object): return self._ffi.buffer(pp[0], res)[:] def x25519_load_public_bytes(self, data): + # When we drop support for CRYPTOGRAPHY_OPENSSL_LESS_THAN_111 we can + # switch this to EVP_PKEY_new_raw_public_key + if len(data) != 32: + raise ValueError("An X25519 public key is 32 bytes long") + evp_pkey = self._create_evp_pkey_gc() res = self._lib.EVP_PKEY_set_type(evp_pkey, self._lib.NID_X25519) backend.openssl_assert(res == 1) @@ -2037,6 +2177,9 @@ class Backend(object): return _X25519PublicKey(self, evp_pkey) def x25519_load_private_bytes(self, data): + # When we drop support for CRYPTOGRAPHY_OPENSSL_LESS_THAN_111 we can + # switch this to EVP_PKEY_new_raw_private_key and drop the + # zeroed_bytearray garbage. # OpenSSL only has facilities for loading PKCS8 formatted private # keys using the algorithm identifiers specified in # https://tools.ietf.org/html/draft-ietf-curdle-pkix-09. @@ -2050,9 +2193,16 @@ class Backend(object): # Of course there's a bit more complexity. In reality OCTET STRING # contains an OCTET STRING of length 32! So the last two bytes here # are \x04\x20, which is an OCTET STRING of length 32. + if len(data) != 32: + raise ValueError("An X25519 private key is 32 bytes long") + pkcs8_prefix = b'0.\x02\x01\x000\x05\x06\x03+en\x04"\x04 ' - bio = self._bytes_to_bio(pkcs8_prefix + data) - evp_pkey = backend._lib.d2i_PrivateKey_bio(bio.bio, self._ffi.NULL) + with self._zeroed_bytearray(48) as ba: + ba[0:16] = pkcs8_prefix + ba[16:] = data + bio = self._bytes_to_bio(ba) + evp_pkey = backend._lib.d2i_PrivateKey_bio(bio.bio, self._ffi.NULL) + self.openssl_assert(evp_pkey != self._ffi.NULL) evp_pkey = self._ffi.gc(evp_pkey, self._lib.EVP_PKEY_free) self.openssl_assert( @@ -2060,14 +2210,10 @@ class Backend(object): ) return _X25519PrivateKey(self, evp_pkey) - def x25519_generate_key(self): - evp_pkey_ctx = self._lib.EVP_PKEY_CTX_new_id( - self._lib.NID_X25519, self._ffi.NULL - ) + def _evp_pkey_keygen_gc(self, nid): + evp_pkey_ctx = self._lib.EVP_PKEY_CTX_new_id(nid, self._ffi.NULL) self.openssl_assert(evp_pkey_ctx != self._ffi.NULL) - evp_pkey_ctx = self._ffi.gc( - evp_pkey_ctx, self._lib.EVP_PKEY_CTX_free - ) + evp_pkey_ctx = self._ffi.gc(evp_pkey_ctx, self._lib.EVP_PKEY_CTX_free) res = self._lib.EVP_PKEY_keygen_init(evp_pkey_ctx) self.openssl_assert(res == 1) evp_ppkey = self._ffi.new("EVP_PKEY **") @@ -2075,18 +2221,143 @@ class Backend(object): self.openssl_assert(res == 1) self.openssl_assert(evp_ppkey[0] != self._ffi.NULL) evp_pkey = self._ffi.gc(evp_ppkey[0], self._lib.EVP_PKEY_free) + return evp_pkey + + def x25519_generate_key(self): + evp_pkey = self._evp_pkey_keygen_gc(self._lib.NID_X25519) return _X25519PrivateKey(self, evp_pkey) def x25519_supported(self): return self._lib.CRYPTOGRAPHY_OPENSSL_110_OR_GREATER + def x448_load_public_bytes(self, data): + if len(data) != 56: + raise ValueError("An X448 public key is 56 bytes long") + + evp_pkey = self._lib.EVP_PKEY_new_raw_public_key( + self._lib.NID_X448, self._ffi.NULL, data, len(data) + ) + self.openssl_assert(evp_pkey != self._ffi.NULL) + evp_pkey = self._ffi.gc(evp_pkey, self._lib.EVP_PKEY_free) + return _X448PublicKey(self, evp_pkey) + + def x448_load_private_bytes(self, data): + if len(data) != 56: + raise ValueError("An X448 private key is 56 bytes long") + + data_ptr = self._ffi.from_buffer(data) + evp_pkey = self._lib.EVP_PKEY_new_raw_private_key( + self._lib.NID_X448, self._ffi.NULL, data_ptr, len(data) + ) + self.openssl_assert(evp_pkey != self._ffi.NULL) + evp_pkey = self._ffi.gc(evp_pkey, self._lib.EVP_PKEY_free) + return _X448PrivateKey(self, evp_pkey) + + def x448_generate_key(self): + evp_pkey = self._evp_pkey_keygen_gc(self._lib.NID_X448) + return _X448PrivateKey(self, evp_pkey) + + def x448_supported(self): + return not self._lib.CRYPTOGRAPHY_OPENSSL_LESS_THAN_111 + + def ed25519_supported(self): + return not self._lib.CRYPTOGRAPHY_OPENSSL_LESS_THAN_111B + + def ed25519_load_public_bytes(self, data): + utils._check_bytes("data", data) + + if len(data) != ed25519._ED25519_KEY_SIZE: + raise ValueError("An Ed25519 public key is 32 bytes long") + + evp_pkey = self._lib.EVP_PKEY_new_raw_public_key( + self._lib.NID_ED25519, self._ffi.NULL, data, len(data) + ) + self.openssl_assert(evp_pkey != self._ffi.NULL) + evp_pkey = self._ffi.gc(evp_pkey, self._lib.EVP_PKEY_free) + + return _Ed25519PublicKey(self, evp_pkey) + + def ed25519_load_private_bytes(self, data): + if len(data) != ed25519._ED25519_KEY_SIZE: + raise ValueError("An Ed25519 private key is 32 bytes long") + + utils._check_byteslike("data", data) + data_ptr = self._ffi.from_buffer(data) + evp_pkey = self._lib.EVP_PKEY_new_raw_private_key( + self._lib.NID_ED25519, self._ffi.NULL, data_ptr, len(data) + ) + self.openssl_assert(evp_pkey != self._ffi.NULL) + evp_pkey = self._ffi.gc(evp_pkey, self._lib.EVP_PKEY_free) + + return _Ed25519PrivateKey(self, evp_pkey) + + def ed25519_generate_key(self): + evp_pkey = self._evp_pkey_keygen_gc(self._lib.NID_ED25519) + return _Ed25519PrivateKey(self, evp_pkey) + + def ed448_supported(self): + return not self._lib.CRYPTOGRAPHY_OPENSSL_LESS_THAN_111B + + def ed448_load_public_bytes(self, data): + utils._check_bytes("data", data) + if len(data) != _ED448_KEY_SIZE: + raise ValueError("An Ed448 public key is 57 bytes long") + + evp_pkey = self._lib.EVP_PKEY_new_raw_public_key( + self._lib.NID_ED448, self._ffi.NULL, data, len(data) + ) + self.openssl_assert(evp_pkey != self._ffi.NULL) + evp_pkey = self._ffi.gc(evp_pkey, self._lib.EVP_PKEY_free) + + return _Ed448PublicKey(self, evp_pkey) + + def ed448_load_private_bytes(self, data): + utils._check_byteslike("data", data) + if len(data) != _ED448_KEY_SIZE: + raise ValueError("An Ed448 private key is 57 bytes long") + + data_ptr = self._ffi.from_buffer(data) + evp_pkey = self._lib.EVP_PKEY_new_raw_private_key( + self._lib.NID_ED448, self._ffi.NULL, data_ptr, len(data) + ) + self.openssl_assert(evp_pkey != self._ffi.NULL) + evp_pkey = self._ffi.gc(evp_pkey, self._lib.EVP_PKEY_free) + + return _Ed448PrivateKey(self, evp_pkey) + + def ed448_generate_key(self): + evp_pkey = self._evp_pkey_keygen_gc(self._lib.NID_ED448) + return _Ed448PrivateKey(self, evp_pkey) + def derive_scrypt(self, key_material, salt, length, n, r, p): buf = self._ffi.new("unsigned char[]", length) + key_material_ptr = self._ffi.from_buffer(key_material) res = self._lib.EVP_PBE_scrypt( - key_material, len(key_material), salt, len(salt), n, r, p, + key_material_ptr, len(key_material), salt, len(salt), n, r, p, scrypt._MEM_LIMIT, buf, length ) - self.openssl_assert(res == 1) + if res != 1: + errors = self._consume_errors() + if not self._lib.CRYPTOGRAPHY_OPENSSL_LESS_THAN_111: + # This error is only added to the stack in 1.1.1+ + self.openssl_assert( + errors[0]._lib_reason_match( + self._lib.ERR_LIB_EVP, + self._lib.ERR_R_MALLOC_FAILURE + ) or + errors[0]._lib_reason_match( + self._lib.ERR_LIB_EVP, + self._lib.EVP_R_MEMORY_LIMIT_EXCEEDED + ) + ) + + # memory required formula explained here: + # https://blog.filippo.io/the-scrypt-parameters/ + min_memory = 128 * n * r // (1024**2) + raise MemoryError( + "Not enough memory to derive key. These parameters require" + " {} MB of memory.".format(min_memory) + ) return self._ffi.buffer(buf)[:] def aead_cipher_supported(self, cipher): @@ -2095,6 +2366,105 @@ class Backend(object): self._lib.EVP_get_cipherbyname(cipher_name) != self._ffi.NULL ) + @contextlib.contextmanager + def _zeroed_bytearray(self, length): + """ + This method creates a bytearray, which we copy data into (hopefully + also from a mutable buffer that can be dynamically erased!), and then + zero when we're done. + """ + ba = bytearray(length) + try: + yield ba + finally: + self._zero_data(ba, length) + + def _zero_data(self, data, length): + # We clear things this way because at the moment we're not + # sure of a better way that can guarantee it overwrites the + # memory of a bytearray and doesn't just replace the underlying char *. + for i in range(length): + data[i] = 0 + + @contextlib.contextmanager + def _zeroed_null_terminated_buf(self, data): + """ + This method takes bytes, which can be a bytestring or a mutable + buffer like a bytearray, and yields a null-terminated version of that + data. This is required because PKCS12_parse doesn't take a length with + its password char * and ffi.from_buffer doesn't provide null + termination. So, to support zeroing the data via bytearray we + need to build this ridiculous construct that copies the memory, but + zeroes it after use. + """ + if data is None: + yield self._ffi.NULL + else: + data_len = len(data) + buf = self._ffi.new("char[]", data_len + 1) + self._ffi.memmove(buf, data, data_len) + try: + yield buf + finally: + # Cast to a uint8_t * so we can assign by integer + self._zero_data(self._ffi.cast("uint8_t *", buf), data_len) + + def load_key_and_certificates_from_pkcs12(self, data, password): + if password is not None: + utils._check_byteslike("password", password) + + bio = self._bytes_to_bio(data) + p12 = self._lib.d2i_PKCS12_bio(bio.bio, self._ffi.NULL) + if p12 == self._ffi.NULL: + self._consume_errors() + raise ValueError("Could not deserialize PKCS12 data") + + p12 = self._ffi.gc(p12, self._lib.PKCS12_free) + evp_pkey_ptr = self._ffi.new("EVP_PKEY **") + x509_ptr = self._ffi.new("X509 **") + sk_x509_ptr = self._ffi.new("Cryptography_STACK_OF_X509 **") + with self._zeroed_null_terminated_buf(password) as password_buf: + res = self._lib.PKCS12_parse( + p12, password_buf, evp_pkey_ptr, x509_ptr, sk_x509_ptr + ) + + if res == 0: + self._consume_errors() + raise ValueError("Invalid password or PKCS12 data") + + cert = None + key = None + additional_certificates = [] + + if evp_pkey_ptr[0] != self._ffi.NULL: + evp_pkey = self._ffi.gc(evp_pkey_ptr[0], self._lib.EVP_PKEY_free) + key = self._evp_pkey_to_private_key(evp_pkey) + + if x509_ptr[0] != self._ffi.NULL: + x509 = self._ffi.gc(x509_ptr[0], self._lib.X509_free) + cert = _Certificate(self, x509) + + if sk_x509_ptr[0] != self._ffi.NULL: + sk_x509 = self._ffi.gc(sk_x509_ptr[0], self._lib.sk_X509_free) + num = self._lib.sk_X509_num(sk_x509_ptr[0]) + for i in range(num): + x509 = self._lib.sk_X509_value(sk_x509, i) + x509 = self._ffi.gc(x509, self._lib.X509_free) + self.openssl_assert(x509 != self._ffi.NULL) + additional_certificates.append(_Certificate(self, x509)) + + return (key, cert, additional_certificates) + + def poly1305_supported(self): + return self._lib.Cryptography_HAS_POLY1305 == 1 + + def create_poly1305_ctx(self, key): + utils._check_byteslike("key", key) + if len(key) != _POLY1305_KEY_SIZE: + raise ValueError("A poly1305 key is 32 bytes long") + + return _Poly1305Context(self, key) + class GetCipherByName(object): def __init__(self, fmt): @@ -2106,7 +2476,7 @@ class GetCipherByName(object): def _get_xts_cipher(backend, cipher, mode): - cipher_name = "aes-{0}-xts".format(cipher.key_size // 2) + cipher_name = "aes-{}-xts".format(cipher.key_size // 2) return backend._lib.EVP_get_cipherbyname(cipher_name.encode("ascii")) diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/ciphers.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/ciphers.py index e0ee06e..94b48f5 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/ciphers.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/ciphers.py @@ -40,7 +40,7 @@ class _CipherContext(object): adapter = registry[type(cipher), type(mode)] except KeyError: raise UnsupportedAlgorithm( - "cipher {0} in {1} mode is not supported " + "cipher {} in {} mode is not supported " "by this backend.".format( cipher.name, mode.name if mode else mode), _Reasons.UNSUPPORTED_CIPHER @@ -48,21 +48,25 @@ class _CipherContext(object): evp_cipher = adapter(self._backend, cipher, mode) if evp_cipher == self._backend._ffi.NULL: - raise UnsupportedAlgorithm( - "cipher {0} in {1} mode is not supported " - "by this backend.".format( - cipher.name, mode.name if mode else mode), - _Reasons.UNSUPPORTED_CIPHER - ) + msg = "cipher {0.name} ".format(cipher) + if mode is not None: + msg += "in {0.name} mode ".format(mode) + msg += ( + "is not supported by this backend (Your version of OpenSSL " + "may be too old. Current version: {}.)" + ).format(self._backend.openssl_version_text()) + raise UnsupportedAlgorithm(msg, _Reasons.UNSUPPORTED_CIPHER) if isinstance(mode, modes.ModeWithInitializationVector): - iv_nonce = mode.initialization_vector + iv_nonce = self._backend._ffi.from_buffer( + mode.initialization_vector + ) elif isinstance(mode, modes.ModeWithTweak): - iv_nonce = mode.tweak + iv_nonce = self._backend._ffi.from_buffer(mode.tweak) elif isinstance(mode, modes.ModeWithNonce): - iv_nonce = mode.nonce + iv_nonce = self._backend._ffi.from_buffer(mode.nonce) elif isinstance(cipher, modes.ModeWithNonce): - iv_nonce = cipher.nonce + iv_nonce = self._backend._ffi.from_buffer(cipher.nonce) else: iv_nonce = self._backend._ffi.NULL # begin init with cipher and operation type @@ -105,7 +109,7 @@ class _CipherContext(object): ctx, self._backend._ffi.NULL, self._backend._ffi.NULL, - cipher.key, + self._backend._ffi.from_buffer(cipher.key), iv_nonce, operation ) @@ -123,7 +127,7 @@ class _CipherContext(object): def update_into(self, data, buf): if len(buf) < (len(data) + self._block_size_bytes - 1): raise ValueError( - "buffer must be at least {0} bytes for this " + "buffer must be at least {} bytes for this " "payload".format(len(data) + self._block_size_bytes - 1) ) @@ -131,8 +135,10 @@ class _CipherContext(object): "unsigned char *", self._backend._ffi.from_buffer(buf) ) outlen = self._backend._ffi.new("int *") - res = self._backend._lib.EVP_CipherUpdate(self._ctx, buf, outlen, - data, len(data)) + res = self._backend._lib.EVP_CipherUpdate( + self._ctx, buf, outlen, + self._backend._ffi.from_buffer(data), len(data) + ) self._backend.openssl_assert(res != 0) return outlen[0] @@ -201,7 +207,7 @@ class _CipherContext(object): ) if len(tag) < self._mode._min_tag_length: raise ValueError( - "Authentication tag must be {0} bytes or longer.".format( + "Authentication tag must be {} bytes or longer.".format( self._mode._min_tag_length) ) res = self._backend._lib.EVP_CIPHER_CTX_ctrl( @@ -215,7 +221,8 @@ class _CipherContext(object): def authenticate_additional_data(self, data): outlen = self._backend._ffi.new("int *") res = self._backend._lib.EVP_CipherUpdate( - self._ctx, self._backend._ffi.NULL, outlen, data, len(data) + self._ctx, self._backend._ffi.NULL, outlen, + self._backend._ffi.from_buffer(data), len(data) ) self._backend.openssl_assert(res != 0) diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/cmac.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/cmac.py index e20f66d..d4d46f5 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/cmac.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/cmac.py @@ -9,11 +9,10 @@ from cryptography import utils from cryptography.exceptions import ( InvalidSignature, UnsupportedAlgorithm, _Reasons ) -from cryptography.hazmat.primitives import constant_time, mac +from cryptography.hazmat.primitives import constant_time from cryptography.hazmat.primitives.ciphers.modes import CBC -@utils.register_interface(mac.MACContext) class _CMACContext(object): def __init__(self, backend, algorithm, ctx=None): if not backend.cmac_algorithm_supported(algorithm): @@ -36,8 +35,9 @@ class _CMACContext(object): self._backend.openssl_assert(ctx != self._backend._ffi.NULL) ctx = self._backend._ffi.gc(ctx, self._backend._lib.CMAC_CTX_free) + key_ptr = self._backend._ffi.from_buffer(self._key) res = self._backend._lib.CMAC_Init( - ctx, self._key, len(self._key), + ctx, key_ptr, len(self._key), evp_cipher, self._backend._ffi.NULL ) self._backend.openssl_assert(res == 1) diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/decode_asn1.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/decode_asn1.py index e06e8cd..47c6c65 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/decode_asn1.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/decode_asn1.py @@ -7,11 +7,10 @@ from __future__ import absolute_import, division, print_function import datetime import ipaddress -import asn1crypto.core - import six from cryptography import x509 +from cryptography.hazmat._der import DERReader, INTEGER, NULL, SEQUENCE from cryptography.x509.extensions import _TLS_FEATURE_TYPE_TO_ENUM from cryptography.x509.name import _ASN1_TYPE_TO_ENUM from cryptography.x509.oid import ( @@ -20,10 +19,6 @@ from cryptography.x509.oid import ( ) -class _Integers(asn1crypto.core.SequenceOf): - _child_spec = asn1crypto.core.Integer - - def _obj2txt(backend, obj): # Set to 80 on the recommendation of # https://www.openssl.org/docs/crypto/OBJ_nid2ln.html#return_values @@ -135,7 +130,7 @@ def _decode_general_name(backend, gn): if "1" in bits[prefix:]: raise ValueError("Invalid netmask") - ip = ipaddress.ip_network(base.exploded + u"/{0}".format(prefix)) + ip = ipaddress.ip_network(base.exploded + u"/{}".format(prefix)) else: ip = ipaddress.ip_address(data) @@ -160,7 +155,7 @@ def _decode_general_name(backend, gn): else: # x400Address or ediPartyName raise x509.UnsupportedGeneralNameType( - "{0} is not a supported type".format( + "{} is not a supported type".format( x509._GENERAL_NAMES.get(gn.type, gn.type) ), gn.type @@ -202,27 +197,32 @@ class _X509ExtensionParser(object): ) if oid in seen_oids: raise x509.DuplicateExtension( - "Duplicate {0} extension found".format(oid), oid + "Duplicate {} extension found".format(oid), oid ) # These OIDs are only supported in OpenSSL 1.1.0+ but we want # to support them in all versions of OpenSSL so we decode them # ourselves. if oid == ExtensionOID.TLS_FEATURE: + # The extension contents are a SEQUENCE OF INTEGERs. data = backend._lib.X509_EXTENSION_get_data(ext) - parsed = _Integers.load(_asn1_string_to_bytes(backend, data)) + data_bytes = _asn1_string_to_bytes(backend, data) + features = DERReader(data_bytes).read_single_element(SEQUENCE) + parsed = [] + while not features.is_empty(): + parsed.append(features.read_element(INTEGER).as_integer()) + # Map the features to their enum value. value = x509.TLSFeature( - [_TLS_FEATURE_TYPE_TO_ENUM[x.native] for x in parsed] + [_TLS_FEATURE_TYPE_TO_ENUM[x] for x in parsed] ) extensions.append(x509.Extension(oid, critical, value)) seen_oids.add(oid) continue elif oid == ExtensionOID.PRECERT_POISON: data = backend._lib.X509_EXTENSION_get_data(ext) - parsed = asn1crypto.core.Null.load( - _asn1_string_to_bytes(backend, data) - ) - assert parsed == asn1crypto.core.Null() + # The contents of the extension must be an ASN.1 NULL. + reader = DERReader(_asn1_string_to_bytes(backend, data)) + reader.read_single_element(NULL).check_empty() extensions.append(x509.Extension( oid, critical, x509.PrecertPoison() )) @@ -245,7 +245,7 @@ class _X509ExtensionParser(object): if ext_data == backend._ffi.NULL: backend._consume_errors() raise ValueError( - "The {0} extension is invalid and can't be " + "The {} extension is invalid and can't be " "parsed".format(oid) ) @@ -379,7 +379,14 @@ def _decode_authority_key_identifier(backend, akid): def _decode_authority_information_access(backend, aia): aia = backend._ffi.cast("Cryptography_STACK_OF_ACCESS_DESCRIPTION *", aia) - aia = backend._ffi.gc(aia, backend._lib.sk_ACCESS_DESCRIPTION_free) + aia = backend._ffi.gc( + aia, + lambda x: backend._lib.sk_ACCESS_DESCRIPTION_pop_free( + x, backend._ffi.addressof( + backend._lib._original_lib, "ACCESS_DESCRIPTION_free" + ) + ) + ) num = backend._lib.sk_ACCESS_DESCRIPTION_num(aia) access_descriptions = [] for i in range(num): @@ -464,6 +471,30 @@ def _decode_general_subtrees(backend, stack_subtrees): return subtrees +def _decode_issuing_dist_point(backend, idp): + idp = backend._ffi.cast("ISSUING_DIST_POINT *", idp) + idp = backend._ffi.gc(idp, backend._lib.ISSUING_DIST_POINT_free) + if idp.distpoint != backend._ffi.NULL: + full_name, relative_name = _decode_distpoint(backend, idp.distpoint) + else: + full_name = None + relative_name = None + + only_user = idp.onlyuser == 255 + only_ca = idp.onlyCA == 255 + indirect_crl = idp.indirectCRL == 255 + only_attr = idp.onlyattr == 255 + if idp.onlysomereasons != backend._ffi.NULL: + only_some_reasons = _decode_reasons(backend, idp.onlysomereasons) + else: + only_some_reasons = None + + return x509.IssuingDistributionPoint( + full_name, relative_name, only_user, only_ca, only_some_reasons, + indirect_crl, only_attr + ) + + def _decode_policy_constraints(backend, pc): pc = backend._ffi.cast("POLICY_CONSTRAINTS *", pc) pc = backend._ffi.gc(pc, backend._lib.POLICY_CONSTRAINTS_free) @@ -674,7 +705,7 @@ def _decode_crl_reason(backend, enum): try: return x509.CRLReason(_CRL_ENTRY_REASON_CODE_TO_ENUM[code]) except KeyError: - raise ValueError("Unsupported reason code: {0}".format(code)) + raise ValueError("Unsupported reason code: {}".format(code)) def _decode_invalidity_date(backend, inv_date): @@ -734,7 +765,7 @@ def _asn1_string_to_utf8(backend, asn1_string): res = backend._lib.ASN1_STRING_to_UTF8(buf, asn1_string) if res == -1: raise ValueError( - "Unsupported ASN1 string type. Type: {0}".format(asn1_string.type) + "Unsupported ASN1 string type. Type: {}".format(asn1_string.type) ) backend.openssl_assert(buf[0] != backend._ffi.NULL) @@ -814,6 +845,8 @@ _CRL_EXTENSION_HANDLERS = { ExtensionOID.AUTHORITY_INFORMATION_ACCESS: ( _decode_authority_information_access ), + ExtensionOID.ISSUING_DISTRIBUTION_POINT: _decode_issuing_dist_point, + ExtensionOID.FRESHEST_CRL: _decode_freshest_crl, } _OCSP_REQ_EXTENSION_HANDLERS = { diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/dsa.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/dsa.py index 48886e4..de61f08 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/dsa.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/dsa.py @@ -211,8 +211,7 @@ class _DSAPublicKey(object): def verifier(self, signature, signature_algorithm): _warn_sign_verify_deprecated() - if not isinstance(signature, bytes): - raise TypeError("signature must be bytes.") + utils._check_bytes("signature", signature) _check_not_prehashed(signature_algorithm) return _DSAVerificationContext( diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/ec.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/ec.py index 69da234..3d8681b 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/ec.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/ec.py @@ -34,7 +34,19 @@ def _ec_key_curve_sn(backend, ec_key): # an error for now. if nid == backend._lib.NID_undef: raise NotImplementedError( - "ECDSA certificates with unnamed curves are unsupported " + "ECDSA keys with unnamed curves are unsupported " + "at this time" + ) + + # This is like the above check, but it also catches the case where you + # explicitly encoded a curve with the same parameters as a named curve. + # Don't do that. + if ( + backend._lib.CRYPTOGRAPHY_OPENSSL_110_OR_GREATER and + backend._lib.EC_GROUP_get_asn1_flag(group) == 0 + ): + raise NotImplementedError( + "ECDSA keys with unnamed curves are unsupported " "at this time" ) @@ -62,7 +74,7 @@ def _sn_to_elliptic_curve(backend, sn): return ec._CURVE_TYPES[sn]() except KeyError: raise UnsupportedAlgorithm( - "{0} is not a supported elliptic curve".format(sn), + "{} is not a supported elliptic curve".format(sn), _Reasons.UNSUPPORTED_ELLIPTIC_CURVE ) @@ -127,12 +139,12 @@ class _ECDSAVerificationContext(object): class _EllipticCurvePrivateKey(object): def __init__(self, backend, ec_key_cdata, evp_pkey): self._backend = backend - _mark_asn1_named_ec_curve(backend, ec_key_cdata) self._ec_key = ec_key_cdata self._evp_pkey = evp_pkey sn = _ec_key_curve_sn(backend, ec_key_cdata) self._curve = _sn_to_elliptic_curve(backend, sn) + _mark_asn1_named_ec_curve(backend, ec_key_cdata) curve = utils.read_only_property("_curve") @@ -229,12 +241,12 @@ class _EllipticCurvePrivateKey(object): class _EllipticCurvePublicKey(object): def __init__(self, backend, ec_key_cdata, evp_pkey): self._backend = backend - _mark_asn1_named_ec_curve(backend, ec_key_cdata) self._ec_key = ec_key_cdata self._evp_pkey = evp_pkey sn = _ec_key_curve_sn(backend, ec_key_cdata) self._curve = _sn_to_elliptic_curve(backend, sn) + _mark_asn1_named_ec_curve(backend, ec_key_cdata) curve = utils.read_only_property("_curve") @@ -244,8 +256,7 @@ class _EllipticCurvePublicKey(object): def verifier(self, signature, signature_algorithm): _warn_sign_verify_deprecated() - if not isinstance(signature, bytes): - raise TypeError("signature must be bytes.") + utils._check_bytes("signature", signature) _check_signature_algorithm(signature_algorithm) _check_not_prehashed(signature_algorithm.algorithm) @@ -276,19 +287,62 @@ class _EllipticCurvePublicKey(object): curve=self._curve ) + def _encode_point(self, format): + if format is serialization.PublicFormat.CompressedPoint: + conversion = self._backend._lib.POINT_CONVERSION_COMPRESSED + else: + assert format is serialization.PublicFormat.UncompressedPoint + conversion = self._backend._lib.POINT_CONVERSION_UNCOMPRESSED + + group = self._backend._lib.EC_KEY_get0_group(self._ec_key) + self._backend.openssl_assert(group != self._backend._ffi.NULL) + point = self._backend._lib.EC_KEY_get0_public_key(self._ec_key) + self._backend.openssl_assert(point != self._backend._ffi.NULL) + with self._backend._tmp_bn_ctx() as bn_ctx: + buflen = self._backend._lib.EC_POINT_point2oct( + group, point, conversion, self._backend._ffi.NULL, 0, bn_ctx + ) + self._backend.openssl_assert(buflen > 0) + buf = self._backend._ffi.new("char[]", buflen) + res = self._backend._lib.EC_POINT_point2oct( + group, point, conversion, buf, buflen, bn_ctx + ) + self._backend.openssl_assert(buflen == res) + + return self._backend._ffi.buffer(buf)[:] + def public_bytes(self, encoding, format): if format is serialization.PublicFormat.PKCS1: raise ValueError( "EC public keys do not support PKCS1 serialization" ) - return self._backend._public_key_bytes( - encoding, - format, - self, - self._evp_pkey, - None - ) + if ( + encoding is serialization.Encoding.X962 or + format is serialization.PublicFormat.CompressedPoint or + format is serialization.PublicFormat.UncompressedPoint + ): + if ( + encoding is not serialization.Encoding.X962 or + format not in ( + serialization.PublicFormat.CompressedPoint, + serialization.PublicFormat.UncompressedPoint + ) + ): + raise ValueError( + "X962 encoding must be used with CompressedPoint or " + "UncompressedPoint format" + ) + + return self._encode_point(format) + else: + return self._backend._public_key_bytes( + encoding, + format, + self, + self._evp_pkey, + None + ) def verify(self, signature, data, signature_algorithm): _check_signature_algorithm(signature_algorithm) diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/ed25519.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/ed25519.py new file mode 100644 index 0000000..f38f11d --- /dev/null +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/ed25519.py @@ -0,0 +1,151 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from cryptography import exceptions, utils +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import ( + Ed25519PrivateKey, Ed25519PublicKey, _ED25519_KEY_SIZE, _ED25519_SIG_SIZE +) + + +@utils.register_interface(Ed25519PublicKey) +class _Ed25519PublicKey(object): + def __init__(self, backend, evp_pkey): + self._backend = backend + self._evp_pkey = evp_pkey + + def public_bytes(self, encoding, format): + if ( + encoding is serialization.Encoding.Raw or + format is serialization.PublicFormat.Raw + ): + if ( + encoding is not serialization.Encoding.Raw or + format is not serialization.PublicFormat.Raw + ): + raise ValueError( + "When using Raw both encoding and format must be Raw" + ) + + return self._raw_public_bytes() + + if ( + encoding in serialization._PEM_DER and + format is not serialization.PublicFormat.SubjectPublicKeyInfo + ): + raise ValueError( + "format must be SubjectPublicKeyInfo when encoding is PEM or " + "DER" + ) + + return self._backend._public_key_bytes( + encoding, format, self, self._evp_pkey, None + ) + + def _raw_public_bytes(self): + buf = self._backend._ffi.new("unsigned char []", _ED25519_KEY_SIZE) + buflen = self._backend._ffi.new("size_t *", _ED25519_KEY_SIZE) + res = self._backend._lib.EVP_PKEY_get_raw_public_key( + self._evp_pkey, buf, buflen + ) + self._backend.openssl_assert(res == 1) + self._backend.openssl_assert(buflen[0] == _ED25519_KEY_SIZE) + return self._backend._ffi.buffer(buf, _ED25519_KEY_SIZE)[:] + + def verify(self, signature, data): + evp_md_ctx = self._backend._lib.Cryptography_EVP_MD_CTX_new() + self._backend.openssl_assert(evp_md_ctx != self._backend._ffi.NULL) + evp_md_ctx = self._backend._ffi.gc( + evp_md_ctx, self._backend._lib.Cryptography_EVP_MD_CTX_free + ) + res = self._backend._lib.EVP_DigestVerifyInit( + evp_md_ctx, self._backend._ffi.NULL, self._backend._ffi.NULL, + self._backend._ffi.NULL, self._evp_pkey + ) + self._backend.openssl_assert(res == 1) + res = self._backend._lib.EVP_DigestVerify( + evp_md_ctx, signature, len(signature), data, len(data) + ) + if res != 1: + self._backend._consume_errors() + raise exceptions.InvalidSignature + + +@utils.register_interface(Ed25519PrivateKey) +class _Ed25519PrivateKey(object): + def __init__(self, backend, evp_pkey): + self._backend = backend + self._evp_pkey = evp_pkey + + def public_key(self): + buf = self._backend._ffi.new("unsigned char []", _ED25519_KEY_SIZE) + buflen = self._backend._ffi.new("size_t *", _ED25519_KEY_SIZE) + res = self._backend._lib.EVP_PKEY_get_raw_public_key( + self._evp_pkey, buf, buflen + ) + self._backend.openssl_assert(res == 1) + self._backend.openssl_assert(buflen[0] == _ED25519_KEY_SIZE) + public_bytes = self._backend._ffi.buffer(buf)[:] + return self._backend.ed25519_load_public_bytes(public_bytes) + + def sign(self, data): + evp_md_ctx = self._backend._lib.Cryptography_EVP_MD_CTX_new() + self._backend.openssl_assert(evp_md_ctx != self._backend._ffi.NULL) + evp_md_ctx = self._backend._ffi.gc( + evp_md_ctx, self._backend._lib.Cryptography_EVP_MD_CTX_free + ) + res = self._backend._lib.EVP_DigestSignInit( + evp_md_ctx, self._backend._ffi.NULL, self._backend._ffi.NULL, + self._backend._ffi.NULL, self._evp_pkey + ) + self._backend.openssl_assert(res == 1) + buf = self._backend._ffi.new("unsigned char[]", _ED25519_SIG_SIZE) + buflen = self._backend._ffi.new("size_t *", len(buf)) + res = self._backend._lib.EVP_DigestSign( + evp_md_ctx, buf, buflen, data, len(data) + ) + self._backend.openssl_assert(res == 1) + self._backend.openssl_assert(buflen[0] == _ED25519_SIG_SIZE) + return self._backend._ffi.buffer(buf, buflen[0])[:] + + def private_bytes(self, encoding, format, encryption_algorithm): + if ( + encoding is serialization.Encoding.Raw or + format is serialization.PublicFormat.Raw + ): + if ( + format is not serialization.PrivateFormat.Raw or + encoding is not serialization.Encoding.Raw or not + isinstance(encryption_algorithm, serialization.NoEncryption) + ): + raise ValueError( + "When using Raw both encoding and format must be Raw " + "and encryption_algorithm must be NoEncryption()" + ) + + return self._raw_private_bytes() + + if ( + encoding in serialization._PEM_DER and + format is not serialization.PrivateFormat.PKCS8 + ): + raise ValueError( + "format must be PKCS8 when encoding is PEM or DER" + ) + + return self._backend._private_key_bytes( + encoding, format, encryption_algorithm, self._evp_pkey, None + ) + + def _raw_private_bytes(self): + buf = self._backend._ffi.new("unsigned char []", _ED25519_KEY_SIZE) + buflen = self._backend._ffi.new("size_t *", _ED25519_KEY_SIZE) + res = self._backend._lib.EVP_PKEY_get_raw_private_key( + self._evp_pkey, buf, buflen + ) + self._backend.openssl_assert(res == 1) + self._backend.openssl_assert(buflen[0] == _ED25519_KEY_SIZE) + return self._backend._ffi.buffer(buf, _ED25519_KEY_SIZE)[:] diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/ed448.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/ed448.py new file mode 100644 index 0000000..f541f05 --- /dev/null +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/ed448.py @@ -0,0 +1,154 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from cryptography import exceptions, utils +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed448 import ( + Ed448PrivateKey, Ed448PublicKey +) + +_ED448_KEY_SIZE = 57 +_ED448_SIG_SIZE = 114 + + +@utils.register_interface(Ed448PublicKey) +class _Ed448PublicKey(object): + def __init__(self, backend, evp_pkey): + self._backend = backend + self._evp_pkey = evp_pkey + + def public_bytes(self, encoding, format): + if ( + encoding is serialization.Encoding.Raw or + format is serialization.PublicFormat.Raw + ): + if ( + encoding is not serialization.Encoding.Raw or + format is not serialization.PublicFormat.Raw + ): + raise ValueError( + "When using Raw both encoding and format must be Raw" + ) + + return self._raw_public_bytes() + + if ( + encoding in serialization._PEM_DER and + format is not serialization.PublicFormat.SubjectPublicKeyInfo + ): + raise ValueError( + "format must be SubjectPublicKeyInfo when encoding is PEM or " + "DER" + ) + + return self._backend._public_key_bytes( + encoding, format, self, self._evp_pkey, None + ) + + def _raw_public_bytes(self): + buf = self._backend._ffi.new("unsigned char []", _ED448_KEY_SIZE) + buflen = self._backend._ffi.new("size_t *", _ED448_KEY_SIZE) + res = self._backend._lib.EVP_PKEY_get_raw_public_key( + self._evp_pkey, buf, buflen + ) + self._backend.openssl_assert(res == 1) + self._backend.openssl_assert(buflen[0] == _ED448_KEY_SIZE) + return self._backend._ffi.buffer(buf, _ED448_KEY_SIZE)[:] + + def verify(self, signature, data): + evp_md_ctx = self._backend._lib.Cryptography_EVP_MD_CTX_new() + self._backend.openssl_assert(evp_md_ctx != self._backend._ffi.NULL) + evp_md_ctx = self._backend._ffi.gc( + evp_md_ctx, self._backend._lib.Cryptography_EVP_MD_CTX_free + ) + res = self._backend._lib.EVP_DigestVerifyInit( + evp_md_ctx, self._backend._ffi.NULL, self._backend._ffi.NULL, + self._backend._ffi.NULL, self._evp_pkey + ) + self._backend.openssl_assert(res == 1) + res = self._backend._lib.EVP_DigestVerify( + evp_md_ctx, signature, len(signature), data, len(data) + ) + if res != 1: + self._backend._consume_errors() + raise exceptions.InvalidSignature + + +@utils.register_interface(Ed448PrivateKey) +class _Ed448PrivateKey(object): + def __init__(self, backend, evp_pkey): + self._backend = backend + self._evp_pkey = evp_pkey + + def public_key(self): + buf = self._backend._ffi.new("unsigned char []", _ED448_KEY_SIZE) + buflen = self._backend._ffi.new("size_t *", _ED448_KEY_SIZE) + res = self._backend._lib.EVP_PKEY_get_raw_public_key( + self._evp_pkey, buf, buflen + ) + self._backend.openssl_assert(res == 1) + self._backend.openssl_assert(buflen[0] == _ED448_KEY_SIZE) + public_bytes = self._backend._ffi.buffer(buf)[:] + return self._backend.ed448_load_public_bytes(public_bytes) + + def sign(self, data): + evp_md_ctx = self._backend._lib.Cryptography_EVP_MD_CTX_new() + self._backend.openssl_assert(evp_md_ctx != self._backend._ffi.NULL) + evp_md_ctx = self._backend._ffi.gc( + evp_md_ctx, self._backend._lib.Cryptography_EVP_MD_CTX_free + ) + res = self._backend._lib.EVP_DigestSignInit( + evp_md_ctx, self._backend._ffi.NULL, self._backend._ffi.NULL, + self._backend._ffi.NULL, self._evp_pkey + ) + self._backend.openssl_assert(res == 1) + buf = self._backend._ffi.new("unsigned char[]", _ED448_SIG_SIZE) + buflen = self._backend._ffi.new("size_t *", len(buf)) + res = self._backend._lib.EVP_DigestSign( + evp_md_ctx, buf, buflen, data, len(data) + ) + self._backend.openssl_assert(res == 1) + self._backend.openssl_assert(buflen[0] == _ED448_SIG_SIZE) + return self._backend._ffi.buffer(buf, buflen[0])[:] + + def private_bytes(self, encoding, format, encryption_algorithm): + if ( + encoding is serialization.Encoding.Raw or + format is serialization.PublicFormat.Raw + ): + if ( + format is not serialization.PrivateFormat.Raw or + encoding is not serialization.Encoding.Raw or not + isinstance(encryption_algorithm, serialization.NoEncryption) + ): + raise ValueError( + "When using Raw both encoding and format must be Raw " + "and encryption_algorithm must be NoEncryption()" + ) + + return self._raw_private_bytes() + + if ( + encoding in serialization._PEM_DER and + format is not serialization.PrivateFormat.PKCS8 + ): + raise ValueError( + "format must be PKCS8 when encoding is PEM or DER" + ) + + return self._backend._private_key_bytes( + encoding, format, encryption_algorithm, self._evp_pkey, None + ) + + def _raw_private_bytes(self): + buf = self._backend._ffi.new("unsigned char []", _ED448_KEY_SIZE) + buflen = self._backend._ffi.new("size_t *", _ED448_KEY_SIZE) + res = self._backend._lib.EVP_PKEY_get_raw_private_key( + self._evp_pkey, buf, buflen + ) + self._backend.openssl_assert(res == 1) + self._backend.openssl_assert(buflen[0] == _ED448_KEY_SIZE) + return self._backend._ffi.buffer(buf, _ED448_KEY_SIZE)[:] diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/encode_asn1.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/encode_asn1.py index 6ff1a9a..ca35f0e 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/encode_asn1.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/encode_asn1.py @@ -114,13 +114,15 @@ def _encode_sk_name_entry(backend, attributes): for attribute in attributes: name_entry = _encode_name_entry(backend, attribute) res = backend._lib.sk_X509_NAME_ENTRY_push(stack, name_entry) - backend.openssl_assert(res == 1) + backend.openssl_assert(res >= 1) return stack def _encode_name_entry(backend, attribute): if attribute._type is _ASN1Type.BMPString: value = attribute.value.encode('utf_16_be') + elif attribute._type is _ASN1Type.UniversalString: + value = attribute.value.encode('utf_32_be') else: value = attribute.value.encode('utf8') @@ -136,6 +138,28 @@ def _encode_crl_number_delta_crl_indicator(backend, ext): return _encode_asn1_int_gc(backend, ext.crl_number) +def _encode_issuing_dist_point(backend, ext): + idp = backend._lib.ISSUING_DIST_POINT_new() + backend.openssl_assert(idp != backend._ffi.NULL) + idp = backend._ffi.gc(idp, backend._lib.ISSUING_DIST_POINT_free) + idp.onlyuser = 255 if ext.only_contains_user_certs else 0 + idp.onlyCA = 255 if ext.only_contains_ca_certs else 0 + idp.indirectCRL = 255 if ext.indirect_crl else 0 + idp.onlyattr = 255 if ext.only_contains_attribute_certs else 0 + if ext.only_some_reasons: + idp.onlysomereasons = _encode_reasonflags( + backend, ext.only_some_reasons + ) + + if ext.full_name: + idp.distpoint = _encode_full_name(backend, ext.full_name) + + if ext.relative_name: + idp.distpoint = _encode_relative_name(backend, ext.relative_name) + + return idp + + def _encode_crl_reason(backend, crl_reason): asn1enum = backend._lib.ASN1_ENUMERATED_new() backend.openssl_assert(asn1enum != backend._ffi.NULL) @@ -323,16 +347,22 @@ def _encode_authority_information_access(backend, authority_info_access): aia = backend._lib.sk_ACCESS_DESCRIPTION_new_null() backend.openssl_assert(aia != backend._ffi.NULL) aia = backend._ffi.gc( - aia, backend._lib.sk_ACCESS_DESCRIPTION_free + aia, + lambda x: backend._lib.sk_ACCESS_DESCRIPTION_pop_free( + x, backend._ffi.addressof( + backend._lib._original_lib, "ACCESS_DESCRIPTION_free" + ) + ) ) for access_description in authority_info_access: ad = backend._lib.ACCESS_DESCRIPTION_new() method = _txt2obj( backend, access_description.access_method.dotted_string ) - gn = _encode_general_name(backend, access_description.access_location) + _encode_general_name_preallocated( + backend, access_description.access_location, ad.location + ) ad.method = method - ad.location = gn res = backend._lib.sk_ACCESS_DESCRIPTION_push(aia, ad) backend.openssl_assert(res >= 1) @@ -363,8 +393,13 @@ def _encode_subject_key_identifier(backend, ski): def _encode_general_name(backend, name): + gn = backend._lib.GENERAL_NAME_new() + _encode_general_name_preallocated(backend, name, gn) + return gn + + +def _encode_general_name_preallocated(backend, name, gn): if isinstance(name, x509.DNSName): - gn = backend._lib.GENERAL_NAME_new() backend.openssl_assert(gn != backend._ffi.NULL) gn.type = backend._lib.GEN_DNS @@ -378,7 +413,6 @@ def _encode_general_name(backend, name): backend.openssl_assert(res == 1) gn.d.dNSName = ia5 elif isinstance(name, x509.RegisteredID): - gn = backend._lib.GENERAL_NAME_new() backend.openssl_assert(gn != backend._ffi.NULL) gn.type = backend._lib.GEN_RID obj = backend._lib.OBJ_txt2obj( @@ -387,13 +421,11 @@ def _encode_general_name(backend, name): backend.openssl_assert(obj != backend._ffi.NULL) gn.d.registeredID = obj elif isinstance(name, x509.DirectoryName): - gn = backend._lib.GENERAL_NAME_new() backend.openssl_assert(gn != backend._ffi.NULL) dir_name = _encode_name(backend, name.value) gn.type = backend._lib.GEN_DIRNAME gn.d.directoryName = dir_name elif isinstance(name, x509.IPAddress): - gn = backend._lib.GENERAL_NAME_new() backend.openssl_assert(gn != backend._ffi.NULL) if isinstance(name.value, ipaddress.IPv4Network): packed = ( @@ -411,7 +443,6 @@ def _encode_general_name(backend, name): gn.type = backend._lib.GEN_IPADD gn.d.iPAddress = ipaddr elif isinstance(name, x509.OtherName): - gn = backend._lib.GENERAL_NAME_new() backend.openssl_assert(gn != backend._ffi.NULL) other_name = backend._lib.OTHERNAME_new() backend.openssl_assert(other_name != backend._ffi.NULL) @@ -434,7 +465,6 @@ def _encode_general_name(backend, name): gn.type = backend._lib.GEN_OTHERNAME gn.d.otherName = other_name elif isinstance(name, x509.RFC822Name): - gn = backend._lib.GENERAL_NAME_new() backend.openssl_assert(gn != backend._ffi.NULL) # ia5strings are supposed to be ITU T.50 but to allow round-tripping # of broken certs that encode utf8 we'll encode utf8 here too. @@ -443,7 +473,6 @@ def _encode_general_name(backend, name): gn.type = backend._lib.GEN_EMAIL gn.d.rfc822Name = asn1_str elif isinstance(name, x509.UniformResourceIdentifier): - gn = backend._lib.GENERAL_NAME_new() backend.openssl_assert(gn != backend._ffi.NULL) # ia5strings are supposed to be ITU T.50 but to allow round-tripping # of broken certs that encode utf8 we'll encode utf8 here too. @@ -453,11 +482,9 @@ def _encode_general_name(backend, name): gn.d.uniformResourceIdentifier = asn1_str else: raise ValueError( - "{0} is an unknown GeneralName type".format(name) + "{} is an unknown GeneralName type".format(name) ) - return gn - def _encode_extended_key_usage(backend, extended_key_usage): eku = backend._lib.sk_ASN1_OBJECT_new_null() @@ -482,6 +509,34 @@ _CRLREASONFLAGS = { } +def _encode_reasonflags(backend, reasons): + bitmask = backend._lib.ASN1_BIT_STRING_new() + backend.openssl_assert(bitmask != backend._ffi.NULL) + for reason in reasons: + res = backend._lib.ASN1_BIT_STRING_set_bit( + bitmask, _CRLREASONFLAGS[reason], 1 + ) + backend.openssl_assert(res == 1) + + return bitmask + + +def _encode_full_name(backend, full_name): + dpn = backend._lib.DIST_POINT_NAME_new() + backend.openssl_assert(dpn != backend._ffi.NULL) + dpn.type = _DISTPOINT_TYPE_FULLNAME + dpn.name.fullname = _encode_general_names(backend, full_name) + return dpn + + +def _encode_relative_name(backend, relative_name): + dpn = backend._lib.DIST_POINT_NAME_new() + backend.openssl_assert(dpn != backend._ffi.NULL) + dpn.type = _DISTPOINT_TYPE_RELATIVENAME + dpn.name.relativename = _encode_sk_name_entry(backend, relative_name) + return dpn + + def _encode_cdps_freshest_crl(backend, cdps): cdp = backend._lib.sk_DIST_POINT_new_null() cdp = backend._ffi.gc(cdp, backend._lib.sk_DIST_POINT_free) @@ -490,30 +545,13 @@ def _encode_cdps_freshest_crl(backend, cdps): backend.openssl_assert(dp != backend._ffi.NULL) if point.reasons: - bitmask = backend._lib.ASN1_BIT_STRING_new() - backend.openssl_assert(bitmask != backend._ffi.NULL) - dp.reasons = bitmask - for reason in point.reasons: - res = backend._lib.ASN1_BIT_STRING_set_bit( - bitmask, _CRLREASONFLAGS[reason], 1 - ) - backend.openssl_assert(res == 1) + dp.reasons = _encode_reasonflags(backend, point.reasons) if point.full_name: - dpn = backend._lib.DIST_POINT_NAME_new() - backend.openssl_assert(dpn != backend._ffi.NULL) - dpn.type = _DISTPOINT_TYPE_FULLNAME - dpn.name.fullname = _encode_general_names(backend, point.full_name) - dp.distpoint = dpn + dp.distpoint = _encode_full_name(backend, point.full_name) if point.relative_name: - dpn = backend._lib.DIST_POINT_NAME_new() - backend.openssl_assert(dpn != backend._ffi.NULL) - dpn.type = _DISTPOINT_TYPE_RELATIVENAME - relativename = _encode_sk_name_entry(backend, point.relative_name) - backend.openssl_assert(relativename != backend._ffi.NULL) - dpn.name.relativename = relativename - dp.distpoint = dpn + dp.distpoint = _encode_relative_name(backend, point.relative_name) if point.crl_issuer: dp.CRLissuer = _encode_general_names(backend, point.crl_issuer) @@ -603,6 +641,8 @@ _CRL_EXTENSION_ENCODE_HANDLERS = { ), ExtensionOID.CRL_NUMBER: _encode_crl_number_delta_crl_indicator, ExtensionOID.DELTA_CRL_INDICATOR: _encode_crl_number_delta_crl_indicator, + ExtensionOID.ISSUING_DISTRIBUTION_POINT: _encode_issuing_dist_point, + ExtensionOID.FRESHEST_CRL: _encode_cdps_freshest_crl, } _CRL_ENTRY_EXTENSION_ENCODE_HANDLERS = { diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/hashes.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/hashes.py index c39f57d..7f9d840 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/hashes.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/hashes.py @@ -25,7 +25,7 @@ class _HashContext(object): evp_md = self._backend._evp_md_from_algorithm(algorithm) if evp_md == self._backend._ffi.NULL: raise UnsupportedAlgorithm( - "{0} is not a supported hash on this backend.".format( + "{} is not a supported hash on this backend.".format( algorithm.name), _Reasons.UNSUPPORTED_HASH ) @@ -47,14 +47,32 @@ class _HashContext(object): return _HashContext(self._backend, self.algorithm, ctx=copied_ctx) def update(self, data): - res = self._backend._lib.EVP_DigestUpdate(self._ctx, data, len(data)) + data_ptr = self._backend._ffi.from_buffer(data) + res = self._backend._lib.EVP_DigestUpdate( + self._ctx, data_ptr, len(data) + ) self._backend.openssl_assert(res != 0) def finalize(self): + if isinstance(self.algorithm, hashes.ExtendableOutputFunction): + # extendable output functions use a different finalize + return self._finalize_xof() + else: + buf = self._backend._ffi.new("unsigned char[]", + self._backend._lib.EVP_MAX_MD_SIZE) + outlen = self._backend._ffi.new("unsigned int *") + res = self._backend._lib.EVP_DigestFinal_ex(self._ctx, buf, outlen) + self._backend.openssl_assert(res != 0) + self._backend.openssl_assert( + outlen[0] == self.algorithm.digest_size + ) + return self._backend._ffi.buffer(buf)[:outlen[0]] + + def _finalize_xof(self): buf = self._backend._ffi.new("unsigned char[]", - self._backend._lib.EVP_MAX_MD_SIZE) - outlen = self._backend._ffi.new("unsigned int *") - res = self._backend._lib.EVP_DigestFinal_ex(self._ctx, buf, outlen) + self.algorithm.digest_size) + res = self._backend._lib.EVP_DigestFinalXOF( + self._ctx, buf, self.algorithm.digest_size + ) self._backend.openssl_assert(res != 0) - self._backend.openssl_assert(outlen[0] == self.algorithm.digest_size) - return self._backend._ffi.buffer(buf)[:outlen[0]] + return self._backend._ffi.buffer(buf)[:self.algorithm.digest_size] diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/hmac.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/hmac.py index 99c43f2..2e09cbc 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/hmac.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/hmac.py @@ -9,10 +9,9 @@ from cryptography import utils from cryptography.exceptions import ( InvalidSignature, UnsupportedAlgorithm, _Reasons ) -from cryptography.hazmat.primitives import constant_time, hashes, mac +from cryptography.hazmat.primitives import constant_time, hashes -@utils.register_interface(mac.MACContext) @utils.register_interface(hashes.HashContext) class _HMACContext(object): def __init__(self, backend, key, algorithm, ctx=None): @@ -28,12 +27,13 @@ class _HMACContext(object): evp_md = self._backend._evp_md_from_algorithm(algorithm) if evp_md == self._backend._ffi.NULL: raise UnsupportedAlgorithm( - "{0} is not a supported hash on this backend".format( + "{} is not a supported hash on this backend".format( algorithm.name), _Reasons.UNSUPPORTED_HASH ) + key_ptr = self._backend._ffi.from_buffer(key) res = self._backend._lib.HMAC_Init_ex( - ctx, key, len(key), evp_md, self._backend._ffi.NULL + ctx, key_ptr, len(key), evp_md, self._backend._ffi.NULL ) self._backend.openssl_assert(res != 0) @@ -55,7 +55,8 @@ class _HMACContext(object): ) def update(self, data): - res = self._backend._lib.HMAC_Update(self._ctx, data, len(data)) + data_ptr = self._backend._ffi.from_buffer(data) + res = self._backend._lib.HMAC_Update(self._ctx, data_ptr, len(data)) self._backend.openssl_assert(res != 0) def finalize(self): diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/ocsp.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/ocsp.py index 32e26a0..7420f65 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/ocsp.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/ocsp.py @@ -82,7 +82,7 @@ def _hash_algorithm(backend, cert_id): return _OIDS_TO_HASH[oid] except KeyError: raise UnsupportedAlgorithm( - "Signature algorithm OID: {0} not recognized".format(oid) + "Signature algorithm OID: {} not recognized".format(oid) ) @@ -126,6 +126,17 @@ class _OCSPResponse(object): oid = _obj2txt(self._backend, alg.algorithm) return x509.ObjectIdentifier(oid) + @property + @_requires_successful_response + def signature_hash_algorithm(self): + oid = self.signature_algorithm_oid + try: + return x509._SIG_OIDS_TO_HASH[oid] + except KeyError: + raise UnsupportedAlgorithm( + "Signature algorithm OID:{} not recognized".format(oid) + ) + @property @_requires_successful_response def signature(self): diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/poly1305.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/poly1305.py new file mode 100644 index 0000000..25448dd --- /dev/null +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/poly1305.py @@ -0,0 +1,60 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + + +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives import constant_time + + +_POLY1305_TAG_SIZE = 16 +_POLY1305_KEY_SIZE = 32 + + +class _Poly1305Context(object): + def __init__(self, backend, key): + self._backend = backend + + key_ptr = self._backend._ffi.from_buffer(key) + # This function copies the key into OpenSSL-owned memory so we don't + # need to retain it ourselves + evp_pkey = self._backend._lib.EVP_PKEY_new_raw_private_key( + self._backend._lib.NID_poly1305, + self._backend._ffi.NULL, key_ptr, len(key) + ) + self._backend.openssl_assert(evp_pkey != self._backend._ffi.NULL) + self._evp_pkey = self._backend._ffi.gc( + evp_pkey, self._backend._lib.EVP_PKEY_free + ) + ctx = self._backend._lib.Cryptography_EVP_MD_CTX_new() + self._backend.openssl_assert(ctx != self._backend._ffi.NULL) + self._ctx = self._backend._ffi.gc( + ctx, self._backend._lib.Cryptography_EVP_MD_CTX_free + ) + res = self._backend._lib.EVP_DigestSignInit( + self._ctx, self._backend._ffi.NULL, self._backend._ffi.NULL, + self._backend._ffi.NULL, self._evp_pkey + ) + self._backend.openssl_assert(res == 1) + + def update(self, data): + data_ptr = self._backend._ffi.from_buffer(data) + res = self._backend._lib.EVP_DigestSignUpdate( + self._ctx, data_ptr, len(data) + ) + self._backend.openssl_assert(res != 0) + + def finalize(self): + buf = self._backend._ffi.new("unsigned char[]", _POLY1305_TAG_SIZE) + outlen = self._backend._ffi.new("size_t *") + res = self._backend._lib.EVP_DigestSignFinal(self._ctx, buf, outlen) + self._backend.openssl_assert(res != 0) + self._backend.openssl_assert(outlen[0] == _POLY1305_TAG_SIZE) + return self._backend._ffi.buffer(buf)[:outlen[0]] + + def verify(self, tag): + mac = self.finalize() + if not constant_time.bytes_eq(mac, tag): + raise InvalidSignature("Value did not match computed tag.") diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/rsa.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/rsa.py index 00f5e37..3e4c2fd 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/rsa.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/rsa.py @@ -59,7 +59,7 @@ def _enc_dec_rsa(backend, key, data, padding): else: raise UnsupportedAlgorithm( - "{0} is not supported by this backend.".format( + "{} is not supported by this backend.".format( padding.name ), _Reasons.UNSUPPORTED_PADDING @@ -127,10 +127,11 @@ def _enc_dec_rsa_pkey_ctx(backend, key, data, padding_enum, padding): def _handle_rsa_enc_dec_error(backend, key): errors = backend._consume_errors() backend.openssl_assert(errors) - assert errors[0].lib == backend._lib.ERR_LIB_RSA + backend.openssl_assert(errors[0].lib == backend._lib.ERR_LIB_RSA) if isinstance(key, _RSAPublicKey): - assert (errors[0].reason == - backend._lib.RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE) + backend.openssl_assert( + errors[0].reason == backend._lib.RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE + ) raise ValueError( "Data too long for key size. Encrypt less data or use a " "larger key size." @@ -148,7 +149,7 @@ def _handle_rsa_enc_dec_error(backend, key): if backend._lib.Cryptography_HAS_RSA_R_PKCS_DECODING_ERROR: decoding_errors.append(backend._lib.RSA_R_PKCS_DECODING_ERROR) - assert errors[0].reason in decoding_errors + backend.openssl_assert(errors[0].reason in decoding_errors) raise ValueError("Decryption failed.") @@ -177,7 +178,7 @@ def _rsa_sig_determine_padding(backend, key, padding, algorithm): padding_enum = backend._lib.RSA_PKCS1_PSS_PADDING else: raise UnsupportedAlgorithm( - "{0} is not supported by this backend.".format(padding.name), + "{} is not supported by this backend.".format(padding.name), _Reasons.UNSUPPORTED_PADDING ) @@ -196,7 +197,7 @@ def _rsa_sig_setup(backend, padding, algorithm, key, data, init_func): if res == 0: backend._consume_errors() raise UnsupportedAlgorithm( - "{0} is not supported by this backend for RSA signing.".format( + "{} is not supported by this backend for RSA signing.".format( algorithm.name ), _Reasons.UNSUPPORTED_HASH @@ -236,17 +237,19 @@ def _rsa_sig_sign(backend, padding, algorithm, private_key, data): pkey_ctx, buf, buflen, data, len(data)) if res != 1: errors = backend._consume_errors() - assert errors[0].lib == backend._lib.ERR_LIB_RSA - reason = None - if (errors[0].reason == - backend._lib.RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE): + backend.openssl_assert(errors[0].lib == backend._lib.ERR_LIB_RSA) + if ( + errors[0].reason == + backend._lib.RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE + ): reason = ("Salt length too long for key size. Try using " "MAX_LENGTH instead.") else: - assert (errors[0].reason == - backend._lib.RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY) + backend.openssl_assert( + errors[0].reason == + backend._lib.RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY + ) reason = "Digest too large for key size. Use a larger key." - assert reason is not None raise ValueError(reason) return backend._ffi.buffer(buf)[:] @@ -434,8 +437,7 @@ class _RSAPublicKey(object): def verifier(self, signature, padding, algorithm): _warn_sign_verify_deprecated() - if not isinstance(signature, bytes): - raise TypeError("signature must be bytes.") + utils._check_bytes("signature", signature) _check_not_prehashed(algorithm) return _RSAVerificationContext( diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/utils.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/utils.py index 05d0fe5..ee472c0 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/utils.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/utils.py @@ -11,6 +11,30 @@ from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.asymmetric.utils import Prehashed +def _evp_pkey_derive(backend, evp_pkey, peer_public_key): + ctx = backend._lib.EVP_PKEY_CTX_new(evp_pkey, backend._ffi.NULL) + backend.openssl_assert(ctx != backend._ffi.NULL) + ctx = backend._ffi.gc(ctx, backend._lib.EVP_PKEY_CTX_free) + res = backend._lib.EVP_PKEY_derive_init(ctx) + backend.openssl_assert(res == 1) + res = backend._lib.EVP_PKEY_derive_set_peer( + ctx, peer_public_key._evp_pkey + ) + backend.openssl_assert(res == 1) + keylen = backend._ffi.new("size_t *") + res = backend._lib.EVP_PKEY_derive(ctx, backend._ffi.NULL, keylen) + backend.openssl_assert(res == 1) + backend.openssl_assert(keylen[0] > 0) + buf = backend._ffi.new("unsigned char[]", keylen[0]) + res = backend._lib.EVP_PKEY_derive(ctx, buf, keylen) + if res != 1: + raise ValueError( + "Null shared key derived from public/private pair." + ) + + return backend._ffi.buffer(buf, keylen[0])[:] + + def _calculate_digest_and_algorithm(backend, data, algorithm): if not isinstance(algorithm, Prehashed): hash_ctx = hashes.Hash(algorithm, backend) @@ -40,6 +64,6 @@ def _warn_sign_verify_deprecated(): warnings.warn( "signer and verifier have been deprecated. Please use sign " "and verify instead.", - utils.PersistentlyDeprecated, + utils.PersistentlyDeprecated2017, stacklevel=3 ) diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/x25519.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/x25519.py index 983ece6..9aab25b 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/x25519.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/x25519.py @@ -4,19 +4,66 @@ from __future__ import absolute_import, division, print_function +import warnings + from cryptography import utils +from cryptography.hazmat.backends.openssl.utils import _evp_pkey_derive +from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric.x25519 import ( X25519PrivateKey, X25519PublicKey ) +_X25519_KEY_SIZE = 32 + + @utils.register_interface(X25519PublicKey) class _X25519PublicKey(object): def __init__(self, backend, evp_pkey): self._backend = backend self._evp_pkey = evp_pkey - def public_bytes(self): + def public_bytes(self, encoding=None, format=None): + if encoding is None or format is None: + if encoding is not None or format is not None: + raise ValueError("Both encoding and format are required") + else: + warnings.warn( + "public_bytes now requires encoding and format arguments. " + "Support for calling without arguments will be removed in " + "cryptography 2.7", + utils.DeprecatedIn25, + ) + encoding = serialization.Encoding.Raw + format = serialization.PublicFormat.Raw + if ( + encoding is serialization.Encoding.Raw or + format is serialization.PublicFormat.Raw + ): + if ( + encoding is not serialization.Encoding.Raw or + format is not serialization.PublicFormat.Raw + ): + raise ValueError( + "When using Raw both encoding and format must be Raw" + ) + + return self._raw_public_bytes() + + if ( + encoding in serialization._PEM_DER and + format is not serialization.PublicFormat.SubjectPublicKeyInfo + ): + raise ValueError( + "format must be SubjectPublicKeyInfo when encoding is PEM or " + "DER" + ) + + return self._backend._public_key_bytes( + encoding, format, self, self._evp_pkey, None + ) + + def _raw_public_bytes(self): ucharpp = self._backend._ffi.new("unsigned char **") res = self._backend._lib.EVP_PKEY_get1_tls_encodedpoint( self._evp_pkey, ucharpp @@ -52,28 +99,51 @@ class _X25519PrivateKey(object): if not isinstance(peer_public_key, X25519PublicKey): raise TypeError("peer_public_key must be X25519PublicKey.") - ctx = self._backend._lib.EVP_PKEY_CTX_new( - self._evp_pkey, self._backend._ffi.NULL + return _evp_pkey_derive( + self._backend, self._evp_pkey, peer_public_key ) - self._backend.openssl_assert(ctx != self._backend._ffi.NULL) - ctx = self._backend._ffi.gc(ctx, self._backend._lib.EVP_PKEY_CTX_free) - res = self._backend._lib.EVP_PKEY_derive_init(ctx) - self._backend.openssl_assert(res == 1) - res = self._backend._lib.EVP_PKEY_derive_set_peer( - ctx, peer_public_key._evp_pkey - ) - self._backend.openssl_assert(res == 1) - keylen = self._backend._ffi.new("size_t *") - res = self._backend._lib.EVP_PKEY_derive( - ctx, self._backend._ffi.NULL, keylen - ) - self._backend.openssl_assert(res == 1) - self._backend.openssl_assert(keylen[0] > 0) - buf = self._backend._ffi.new("unsigned char[]", keylen[0]) - res = self._backend._lib.EVP_PKEY_derive(ctx, buf, keylen) - if res != 1: + + def private_bytes(self, encoding, format, encryption_algorithm): + if ( + encoding is serialization.Encoding.Raw or + format is serialization.PublicFormat.Raw + ): + if ( + format is not serialization.PrivateFormat.Raw or + encoding is not serialization.Encoding.Raw or not + isinstance(encryption_algorithm, serialization.NoEncryption) + ): + raise ValueError( + "When using Raw both encoding and format must be Raw " + "and encryption_algorithm must be NoEncryption()" + ) + + return self._raw_private_bytes() + + if ( + encoding in serialization._PEM_DER and + format is not serialization.PrivateFormat.PKCS8 + ): raise ValueError( - "Null shared key derived from public/private pair." + "format must be PKCS8 when encoding is PEM or DER" ) - return self._backend._ffi.buffer(buf, keylen[0])[:] + return self._backend._private_key_bytes( + encoding, format, encryption_algorithm, self._evp_pkey, None + ) + + def _raw_private_bytes(self): + # When we drop support for CRYPTOGRAPHY_OPENSSL_LESS_THAN_111 we can + # switch this to EVP_PKEY_new_raw_private_key + # The trick we use here is serializing to a PKCS8 key and just + # using the last 32 bytes, which is the key itself. + bio = self._backend._create_mem_bio_gc() + res = self._backend._lib.i2d_PKCS8PrivateKey_bio( + bio, self._evp_pkey, + self._backend._ffi.NULL, self._backend._ffi.NULL, + 0, self._backend._ffi.NULL, self._backend._ffi.NULL + ) + self._backend.openssl_assert(res == 1) + pkcs8 = self._backend._read_mem_bio(bio) + self._backend.openssl_assert(len(pkcs8) == 48) + return pkcs8[-_X25519_KEY_SIZE:] diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/x448.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/x448.py new file mode 100644 index 0000000..fe0dcd9 --- /dev/null +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/x448.py @@ -0,0 +1,123 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from cryptography import utils +from cryptography.hazmat.backends.openssl.utils import _evp_pkey_derive +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.x448 import ( + X448PrivateKey, X448PublicKey +) + +_X448_KEY_SIZE = 56 + + +@utils.register_interface(X448PublicKey) +class _X448PublicKey(object): + def __init__(self, backend, evp_pkey): + self._backend = backend + self._evp_pkey = evp_pkey + + def public_bytes(self, encoding, format): + if ( + encoding is serialization.Encoding.Raw or + format is serialization.PublicFormat.Raw + ): + if ( + encoding is not serialization.Encoding.Raw or + format is not serialization.PublicFormat.Raw + ): + raise ValueError( + "When using Raw both encoding and format must be Raw" + ) + + return self._raw_public_bytes() + + if ( + encoding in serialization._PEM_DER and + format is not serialization.PublicFormat.SubjectPublicKeyInfo + ): + raise ValueError( + "format must be SubjectPublicKeyInfo when encoding is PEM or " + "DER" + ) + + return self._backend._public_key_bytes( + encoding, format, self, self._evp_pkey, None + ) + + def _raw_public_bytes(self): + buf = self._backend._ffi.new("unsigned char []", _X448_KEY_SIZE) + buflen = self._backend._ffi.new("size_t *", _X448_KEY_SIZE) + res = self._backend._lib.EVP_PKEY_get_raw_public_key( + self._evp_pkey, buf, buflen + ) + self._backend.openssl_assert(res == 1) + self._backend.openssl_assert(buflen[0] == _X448_KEY_SIZE) + return self._backend._ffi.buffer(buf, _X448_KEY_SIZE)[:] + + +@utils.register_interface(X448PrivateKey) +class _X448PrivateKey(object): + def __init__(self, backend, evp_pkey): + self._backend = backend + self._evp_pkey = evp_pkey + + def public_key(self): + buf = self._backend._ffi.new("unsigned char []", _X448_KEY_SIZE) + buflen = self._backend._ffi.new("size_t *", _X448_KEY_SIZE) + res = self._backend._lib.EVP_PKEY_get_raw_public_key( + self._evp_pkey, buf, buflen + ) + self._backend.openssl_assert(res == 1) + self._backend.openssl_assert(buflen[0] == _X448_KEY_SIZE) + return self._backend.x448_load_public_bytes(buf) + + def exchange(self, peer_public_key): + if not isinstance(peer_public_key, X448PublicKey): + raise TypeError("peer_public_key must be X448PublicKey.") + + return _evp_pkey_derive( + self._backend, self._evp_pkey, peer_public_key + ) + + def private_bytes(self, encoding, format, encryption_algorithm): + if ( + encoding is serialization.Encoding.Raw or + format is serialization.PublicFormat.Raw + ): + if ( + format is not serialization.PrivateFormat.Raw or + encoding is not serialization.Encoding.Raw or not + isinstance(encryption_algorithm, serialization.NoEncryption) + ): + raise ValueError( + "When using Raw both encoding and format must be Raw " + "and encryption_algorithm must be NoEncryption()" + ) + + return self._raw_private_bytes() + + if ( + encoding in serialization._PEM_DER and + format is not serialization.PrivateFormat.PKCS8 + ): + raise ValueError( + "format must be PKCS8 when encoding is PEM or DER" + ) + + return self._backend._private_key_bytes( + encoding, format, encryption_algorithm, self._evp_pkey, None + ) + + def _raw_private_bytes(self): + buf = self._backend._ffi.new("unsigned char []", _X448_KEY_SIZE) + buflen = self._backend._ffi.new("size_t *", _X448_KEY_SIZE) + res = self._backend._lib.EVP_PKEY_get_raw_private_key( + self._evp_pkey, buf, buflen + ) + self._backend.openssl_assert(res == 1) + self._backend.openssl_assert(buflen[0] == _X448_KEY_SIZE) + return self._backend._ffi.buffer(buf, _X448_KEY_SIZE)[:] diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/x509.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/x509.py index ac1838c..efbb179 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/x509.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/backends/openssl/x509.py @@ -6,7 +6,6 @@ from __future__ import absolute_import, division, print_function import datetime import operator -import warnings from cryptography import utils, x509 from cryptography.exceptions import UnsupportedAlgorithm @@ -30,7 +29,7 @@ class _Certificate(object): self._x509 = x509 def __repr__(self): - return "".format(self.subject) + return "".format(self.subject) def __eq__(self, other): if not isinstance(other, x509.Certificate): @@ -59,18 +58,9 @@ class _Certificate(object): return x509.Version.v3 else: raise x509.InvalidVersion( - "{0} is not a valid X509 version".format(version), version + "{} is not a valid X509 version".format(version), version ) - @property - def serial(self): - warnings.warn( - "Certificate serial is deprecated, use serial_number instead.", - utils.PersistentlyDeprecated, - stacklevel=2 - ) - return self.serial_number - @property def serial_number(self): asn1_int = self._backend._lib.X509_get_serialNumber(self._x509) @@ -90,12 +80,12 @@ class _Certificate(object): @property def not_valid_before(self): - asn1_time = self._backend._lib.X509_get_notBefore(self._x509) + asn1_time = self._backend._lib.X509_getm_notBefore(self._x509) return _parse_asn1_time(self._backend, asn1_time) @property def not_valid_after(self): - asn1_time = self._backend._lib.X509_get_notAfter(self._x509) + asn1_time = self._backend._lib.X509_getm_notAfter(self._x509) return _parse_asn1_time(self._backend, asn1_time) @property @@ -117,7 +107,7 @@ class _Certificate(object): return x509._SIG_OIDS_TO_HASH[oid] except KeyError: raise UnsupportedAlgorithm( - "Signature algorithm OID:{0} not recognized".format(oid) + "Signature algorithm OID:{} not recognized".format(oid) ) @property @@ -271,7 +261,7 @@ class _CertificateRevocationList(object): return x509._SIG_OIDS_TO_HASH[oid] except KeyError: raise UnsupportedAlgorithm( - "Signature algorithm OID:{0} not recognized".format(oid) + "Signature algorithm OID:{} not recognized".format(oid) ) @property @@ -423,7 +413,7 @@ class _CertificateSigningRequest(object): return x509._SIG_OIDS_TO_HASH[oid] except KeyError: raise UnsupportedAlgorithm( - "Signature algorithm OID:{0} not recognized".format(oid) + "Signature algorithm OID:{} not recognized".format(oid) ) @property diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/bindings/_constant_time.abi3.so b/server/www/packages/packages-linux/x64/cryptography/hazmat/bindings/_constant_time.abi3.so index 464be7e..4a658ac 100755 Binary files a/server/www/packages/packages-linux/x64/cryptography/hazmat/bindings/_constant_time.abi3.so and b/server/www/packages/packages-linux/x64/cryptography/hazmat/bindings/_constant_time.abi3.so differ diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/bindings/_openssl.abi3.so b/server/www/packages/packages-linux/x64/cryptography/hazmat/bindings/_openssl.abi3.so index 99b5aaf..46bac0b 100755 Binary files a/server/www/packages/packages-linux/x64/cryptography/hazmat/bindings/_openssl.abi3.so and b/server/www/packages/packages-linux/x64/cryptography/hazmat/bindings/_openssl.abi3.so differ diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/bindings/_padding.abi3.so b/server/www/packages/packages-linux/x64/cryptography/hazmat/bindings/_padding.abi3.so index e5304d2..7fe4749 100755 Binary files a/server/www/packages/packages-linux/x64/cryptography/hazmat/bindings/_padding.abi3.so and b/server/www/packages/packages-linux/x64/cryptography/hazmat/bindings/_padding.abi3.so differ diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/bindings/openssl/_conditional.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/bindings/openssl/_conditional.py index 219f163..a39bb66 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/bindings/openssl/_conditional.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/bindings/openssl/_conditional.py @@ -150,11 +150,6 @@ def cryptography_has_tls_st(): def cryptography_has_locking_callbacks(): return [ - "CRYPTO_LOCK", - "CRYPTO_UNLOCK", - "CRYPTO_READ", - "CRYPTO_LOCK_SSL", - "CRYPTO_lock", "Cryptography_setup_ssl_threads", ] @@ -224,9 +219,31 @@ def cryptography_has_x25519(): ] +def cryptography_has_x448(): + return [ + "EVP_PKEY_X448", + "NID_X448", + ] + + +def cryptography_has_ed448(): + return [ + "EVP_PKEY_ED448", + "NID_ED448", + ] + + def cryptography_has_ed25519(): return [ "NID_ED25519", + "EVP_PKEY_ED25519", + ] + + +def cryptography_has_poly1305(): + return [ + "NID_poly1305", + "EVP_PKEY_POLY1305", ] @@ -237,6 +254,12 @@ def cryptography_has_oneshot_evp_digest_sign_verify(): ] +def cryptography_has_evp_digestfinal_xof(): + return [ + "EVP_DigestFinalXOF", + ] + + def cryptography_has_evp_pkey_get_set_tls_encodedpoint(): return [ "EVP_PKEY_get1_tls_encodedpoint", @@ -246,7 +269,7 @@ def cryptography_has_evp_pkey_get_set_tls_encodedpoint(): def cryptography_has_fips(): return [ - "FIPS_set_mode", + "FIPS_mode_set", "FIPS_mode", ] @@ -298,6 +321,46 @@ def cryptography_has_tlsv13(): "SSL_verify_client_post_handshake", "SSL_CTX_set_post_handshake_auth", "SSL_set_post_handshake_auth", + "SSL_SESSION_get_max_early_data", + "SSL_write_early_data", + "SSL_read_early_data", + "SSL_CTX_set_max_early_data", + ] + + +def cryptography_has_raw_key(): + return [ + "EVP_PKEY_new_raw_private_key", + "EVP_PKEY_new_raw_public_key", + "EVP_PKEY_get_raw_private_key", + "EVP_PKEY_get_raw_public_key", + ] + + +def cryptography_has_evp_r_memory_limit_exceeded(): + return [ + "EVP_R_MEMORY_LIMIT_EXCEEDED", + ] + + +def cryptography_has_engine(): + return [ + "ENGINE_by_id", + "ENGINE_init", + "ENGINE_finish", + "ENGINE_get_default_RAND", + "ENGINE_set_default_RAND", + "ENGINE_unregister_RAND", + "ENGINE_ctrl_cmd", + "ENGINE_free", + "ENGINE_get_name", + "Cryptography_add_osrandom_engine", + ] + + +def cryptography_has_verified_chain(): + return [ + "SSL_get0_verified_chain", ] @@ -349,7 +412,10 @@ CONDITIONAL_NAMES = { cryptography_has_x509_store_ctx_get_issuer ), "Cryptography_HAS_X25519": cryptography_has_x25519, + "Cryptography_HAS_X448": cryptography_has_x448, + "Cryptography_HAS_ED448": cryptography_has_ed448, "Cryptography_HAS_ED25519": cryptography_has_ed25519, + "Cryptography_HAS_POLY1305": cryptography_has_poly1305, "Cryptography_HAS_ONESHOT_EVP_DIGEST_SIGN_VERIFY": ( cryptography_has_oneshot_evp_digest_sign_verify ), @@ -363,4 +429,13 @@ CONDITIONAL_NAMES = { "Cryptography_HAS_OPENSSL_CLEANUP": cryptography_has_openssl_cleanup, "Cryptography_HAS_CIPHER_DETAILS": cryptography_has_cipher_details, "Cryptography_HAS_TLSv1_3": cryptography_has_tlsv13, + "Cryptography_HAS_RAW_KEY": cryptography_has_raw_key, + "Cryptography_HAS_EVP_DIGESTFINAL_XOF": ( + cryptography_has_evp_digestfinal_xof + ), + "Cryptography_HAS_EVP_R_MEMORY_LIMIT_EXCEEDED": ( + cryptography_has_evp_r_memory_limit_exceeded + ), + "Cryptography_HAS_ENGINE": cryptography_has_engine, + "Cryptography_HAS_VERIFIED_CHAIN": cryptography_has_verified_chain, } diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/bindings/openssl/binding.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/bindings/openssl/binding.py index 0824ea8..9740516 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/bindings/openssl/binding.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/bindings/openssl/binding.py @@ -9,6 +9,7 @@ import threading import types import warnings +import cryptography from cryptography import utils from cryptography.exceptions import InternalError from cryptography.hazmat.bindings._openssl import ffi, lib @@ -114,10 +115,9 @@ class Binding(object): # reliably clear the error queue. Once we clear it here we will # error on any subsequent unexpected item in the stack. cls.lib.ERR_clear_error() - cls._osrandom_engine_id = cls.lib.Cryptography_osrandom_engine_id - cls._osrandom_engine_name = cls.lib.Cryptography_osrandom_engine_name - result = cls.lib.Cryptography_add_osrandom_engine() - _openssl_assert(cls.lib, result in (1, 2)) + if cls.lib.Cryptography_HAS_ENGINE: + result = cls.lib.Cryptography_add_osrandom_engine() + _openssl_assert(cls.lib, result in (1, 2)) @classmethod def _ensure_ffi_initialized(cls): @@ -158,12 +158,35 @@ def _verify_openssl_version(lib): ): warnings.warn( "OpenSSL version 1.0.1 is no longer supported by the OpenSSL " - "project, please upgrade. A future version of cryptography will " + "project, please upgrade. The next version of cryptography will " "drop support for it.", utils.CryptographyDeprecationWarning ) +def _verify_package_version(version): + # Occasionally we run into situations where the version of the Python + # package does not match the version of the shared object that is loaded. + # This may occur in environments where multiple versions of cryptography + # are installed and available in the python path. To avoid errors cropping + # up later this code checks that the currently imported package and the + # shared object that were loaded have the same version and raise an + # ImportError if they do not + so_package_version = ffi.string(lib.CRYPTOGRAPHY_PACKAGE_VERSION) + if version.encode("ascii") != so_package_version: + raise ImportError( + "The version of cryptography does not match the loaded " + "shared object. This can happen if you have multiple copies of " + "cryptography installed in your Python path. Please try creating " + "a new virtual environment to resolve this issue. " + "Loaded python version: {}, shared object version: {}".format( + version, so_package_version + ) + ) + + +_verify_package_version(cryptography.__version__) + # OpenSSL is not thread safe until the locks are initialized. We call this # method in module scope so that it executes with the import lock. On # Pythons < 3.4 this import lock is a global lock, which can prevent a race diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/asymmetric/ec.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/asymmetric/ec.py index 431ecb7..529391f 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/asymmetric/ec.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/asymmetric/ec.py @@ -5,6 +5,7 @@ from __future__ import absolute_import, division, print_function import abc +import warnings import six @@ -19,6 +20,19 @@ class EllipticCurveOID(object): SECP256R1 = ObjectIdentifier("1.2.840.10045.3.1.7") SECP384R1 = ObjectIdentifier("1.3.132.0.34") SECP521R1 = ObjectIdentifier("1.3.132.0.35") + BRAINPOOLP256R1 = ObjectIdentifier("1.3.36.3.3.2.8.1.1.7") + BRAINPOOLP384R1 = ObjectIdentifier("1.3.36.3.3.2.8.1.1.11") + BRAINPOOLP512R1 = ObjectIdentifier("1.3.36.3.3.2.8.1.1.13") + SECT163K1 = ObjectIdentifier("1.3.132.0.1") + SECT163R2 = ObjectIdentifier("1.3.132.0.15") + SECT233K1 = ObjectIdentifier("1.3.132.0.26") + SECT233R1 = ObjectIdentifier("1.3.132.0.27") + SECT283K1 = ObjectIdentifier("1.3.132.0.16") + SECT283R1 = ObjectIdentifier("1.3.132.0.17") + SECT409K1 = ObjectIdentifier("1.3.132.0.36") + SECT409R1 = ObjectIdentifier("1.3.132.0.37") + SECT571K1 = ObjectIdentifier("1.3.132.0.38") + SECT571R1 = ObjectIdentifier("1.3.132.0.39") @six.add_metaclass(abc.ABCMeta) @@ -138,6 +152,22 @@ class EllipticCurvePublicKey(object): Verifies the signature of the data. """ + @classmethod + def from_encoded_point(cls, curve, data): + utils._check_bytes("data", data) + + if not isinstance(curve, EllipticCurve): + raise TypeError("curve must be an EllipticCurve instance") + + if len(data) == 0: + raise ValueError("data must not be an empty byte string") + + if six.indexbytes(data, 0) not in [0x02, 0x03, 0x04]: + raise ValueError("Unsupported elliptic curve point type") + + from cryptography.hazmat.backends.openssl.backend import backend + return backend.load_elliptic_curve_public_bytes(curve, data) + EllipticCurvePublicKeyWithSerialization = EllipticCurvePublicKey @@ -329,6 +359,14 @@ class EllipticCurvePublicNumbers(object): return backend.load_elliptic_curve_public_numbers(self) def encode_point(self): + warnings.warn( + "encode_point has been deprecated on EllipticCurvePublicNumbers" + " and will be removed in a future version. Please use " + "EllipticCurvePublicKey.public_bytes to obtain both " + "compressed and uncompressed point encoding.", + utils.DeprecatedIn25, + stacklevel=2, + ) # key_size is in bits. Convert to bytes and round up byte_length = (self.curve.key_size + 7) // 8 return ( @@ -341,6 +379,14 @@ class EllipticCurvePublicNumbers(object): if not isinstance(curve, EllipticCurve): raise TypeError("curve must be an EllipticCurve instance") + warnings.warn( + "Support for unsafe construction of public numbers from " + "encoded data will be removed in a future version. " + "Please use EllipticCurvePublicKey.from_encoded_point", + utils.DeprecatedIn25, + stacklevel=2, + ) + if data.startswith(b'\x04'): # key_size is in bits. Convert to bytes and round up byte_length = (curve.key_size + 7) // 8 @@ -419,3 +465,36 @@ class EllipticCurvePrivateNumbers(object): class ECDH(object): pass + + +_OID_TO_CURVE = { + EllipticCurveOID.SECP192R1: SECP192R1, + EllipticCurveOID.SECP224R1: SECP224R1, + EllipticCurveOID.SECP256K1: SECP256K1, + EllipticCurveOID.SECP256R1: SECP256R1, + EllipticCurveOID.SECP384R1: SECP384R1, + EllipticCurveOID.SECP521R1: SECP521R1, + EllipticCurveOID.BRAINPOOLP256R1: BrainpoolP256R1, + EllipticCurveOID.BRAINPOOLP384R1: BrainpoolP384R1, + EllipticCurveOID.BRAINPOOLP512R1: BrainpoolP512R1, + EllipticCurveOID.SECT163K1: SECT163K1, + EllipticCurveOID.SECT163R2: SECT163R2, + EllipticCurveOID.SECT233K1: SECT233K1, + EllipticCurveOID.SECT233R1: SECT233R1, + EllipticCurveOID.SECT283K1: SECT283K1, + EllipticCurveOID.SECT283R1: SECT283R1, + EllipticCurveOID.SECT409K1: SECT409K1, + EllipticCurveOID.SECT409R1: SECT409R1, + EllipticCurveOID.SECT571K1: SECT571K1, + EllipticCurveOID.SECT571R1: SECT571R1, +} + + +def get_curve_for_oid(oid): + try: + return _OID_TO_CURVE[oid] + except KeyError: + raise LookupError( + "The provided object identifier has no matching elliptic " + "curve class" + ) diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/asymmetric/ed25519.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/asymmetric/ed25519.py new file mode 100644 index 0000000..d89445f --- /dev/null +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/asymmetric/ed25519.py @@ -0,0 +1,84 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import abc + +import six + +from cryptography.exceptions import UnsupportedAlgorithm, _Reasons + + +_ED25519_KEY_SIZE = 32 +_ED25519_SIG_SIZE = 64 + + +@six.add_metaclass(abc.ABCMeta) +class Ed25519PublicKey(object): + @classmethod + def from_public_bytes(cls, data): + from cryptography.hazmat.backends.openssl.backend import backend + if not backend.ed25519_supported(): + raise UnsupportedAlgorithm( + "ed25519 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM + ) + + return backend.ed25519_load_public_bytes(data) + + @abc.abstractmethod + def public_bytes(self, encoding, format): + """ + The serialized bytes of the public key. + """ + + @abc.abstractmethod + def verify(self, signature, data): + """ + Verify the signature. + """ + + +@six.add_metaclass(abc.ABCMeta) +class Ed25519PrivateKey(object): + @classmethod + def generate(cls): + from cryptography.hazmat.backends.openssl.backend import backend + if not backend.ed25519_supported(): + raise UnsupportedAlgorithm( + "ed25519 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM + ) + + return backend.ed25519_generate_key() + + @classmethod + def from_private_bytes(cls, data): + from cryptography.hazmat.backends.openssl.backend import backend + if not backend.ed25519_supported(): + raise UnsupportedAlgorithm( + "ed25519 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM + ) + + return backend.ed25519_load_private_bytes(data) + + @abc.abstractmethod + def public_key(self): + """ + The Ed25519PublicKey derived from the private key. + """ + + @abc.abstractmethod + def private_bytes(self, encoding, format, encryption_algorithm): + """ + The serialized bytes of the private key. + """ + + @abc.abstractmethod + def sign(self, data): + """ + Signs the data. + """ diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/asymmetric/ed448.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/asymmetric/ed448.py new file mode 100644 index 0000000..939157a --- /dev/null +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/asymmetric/ed448.py @@ -0,0 +1,79 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import abc + +import six + +from cryptography.exceptions import UnsupportedAlgorithm, _Reasons + + +@six.add_metaclass(abc.ABCMeta) +class Ed448PublicKey(object): + @classmethod + def from_public_bytes(cls, data): + from cryptography.hazmat.backends.openssl.backend import backend + if not backend.ed448_supported(): + raise UnsupportedAlgorithm( + "ed448 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM + ) + + return backend.ed448_load_public_bytes(data) + + @abc.abstractmethod + def public_bytes(self, encoding, format): + """ + The serialized bytes of the public key. + """ + + @abc.abstractmethod + def verify(self, signature, data): + """ + Verify the signature. + """ + + +@six.add_metaclass(abc.ABCMeta) +class Ed448PrivateKey(object): + @classmethod + def generate(cls): + from cryptography.hazmat.backends.openssl.backend import backend + if not backend.ed448_supported(): + raise UnsupportedAlgorithm( + "ed448 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM + ) + return backend.ed448_generate_key() + + @classmethod + def from_private_bytes(cls, data): + from cryptography.hazmat.backends.openssl.backend import backend + if not backend.ed448_supported(): + raise UnsupportedAlgorithm( + "ed448 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM + ) + + return backend.ed448_load_private_bytes(data) + + @abc.abstractmethod + def public_key(self): + """ + The Ed448PublicKey derived from the private key. + """ + + @abc.abstractmethod + def sign(self, data): + """ + Signs the data. + """ + + @abc.abstractmethod + def private_bytes(self, encoding, format, encryption_algorithm): + """ + The serialized bytes of the private key. + """ diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/asymmetric/utils.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/asymmetric/utils.py index ef1e7eb..14d2abe 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/asymmetric/utils.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/asymmetric/utils.py @@ -4,49 +4,26 @@ from __future__ import absolute_import, division, print_function -import warnings - -from asn1crypto.algos import DSASignature - -import six - from cryptography import utils +from cryptography.hazmat._der import ( + DERReader, INTEGER, SEQUENCE, encode_der, encode_der_integer +) from cryptography.hazmat.primitives import hashes -def decode_rfc6979_signature(signature): - warnings.warn( - "decode_rfc6979_signature is deprecated and will " - "be removed in a future version, use decode_dss_signature instead.", - utils.PersistentlyDeprecated, - stacklevel=2 - ) - return decode_dss_signature(signature) - - def decode_dss_signature(signature): - data = DSASignature.load(signature, strict=True).native - return data['r'], data['s'] - - -def encode_rfc6979_signature(r, s): - warnings.warn( - "encode_rfc6979_signature is deprecated and will " - "be removed in a future version, use encode_dss_signature instead.", - utils.PersistentlyDeprecated, - stacklevel=2 - ) - return encode_dss_signature(r, s) + with DERReader(signature).read_single_element(SEQUENCE) as seq: + r = seq.read_element(INTEGER).as_integer() + s = seq.read_element(INTEGER).as_integer() + return r, s def encode_dss_signature(r, s): - if ( - not isinstance(r, six.integer_types) or - not isinstance(s, six.integer_types) - ): - raise ValueError("Both r and s must be integers") - - return DSASignature({'r': r, 's': s}).dump() + return encode_der( + SEQUENCE, + encode_der(INTEGER, encode_der_integer(r)), + encode_der(INTEGER, encode_der_integer(s)), + ) class Prehashed(object): diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/asymmetric/x25519.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/asymmetric/x25519.py index 5c4652a..4e8badf 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/asymmetric/x25519.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/asymmetric/x25519.py @@ -21,11 +21,14 @@ class X25519PublicKey(object): "X25519 is not supported by this version of OpenSSL.", _Reasons.UNSUPPORTED_EXCHANGE_ALGORITHM ) + return backend.x25519_load_public_bytes(data) @abc.abstractmethod - def public_bytes(self): - pass + def public_bytes(self, encoding=None, format=None): + """ + The serialized bytes of the public key. + """ @six.add_metaclass(abc.ABCMeta) @@ -41,14 +44,30 @@ class X25519PrivateKey(object): return backend.x25519_generate_key() @classmethod - def _from_private_bytes(cls, data): + def from_private_bytes(cls, data): from cryptography.hazmat.backends.openssl.backend import backend + if not backend.x25519_supported(): + raise UnsupportedAlgorithm( + "X25519 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_EXCHANGE_ALGORITHM + ) + return backend.x25519_load_private_bytes(data) @abc.abstractmethod def public_key(self): - pass + """ + The serialized bytes of the public key. + """ + + @abc.abstractmethod + def private_bytes(self, encoding, format, encryption_algorithm): + """ + The serialized bytes of the private key. + """ @abc.abstractmethod def exchange(self, peer_public_key): - pass + """ + Performs a key exchange operation using the provided peer's public key. + """ diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/asymmetric/x448.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/asymmetric/x448.py new file mode 100644 index 0000000..475e678 --- /dev/null +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/asymmetric/x448.py @@ -0,0 +1,73 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import abc + +import six + +from cryptography.exceptions import UnsupportedAlgorithm, _Reasons + + +@six.add_metaclass(abc.ABCMeta) +class X448PublicKey(object): + @classmethod + def from_public_bytes(cls, data): + from cryptography.hazmat.backends.openssl.backend import backend + if not backend.x448_supported(): + raise UnsupportedAlgorithm( + "X448 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_EXCHANGE_ALGORITHM + ) + + return backend.x448_load_public_bytes(data) + + @abc.abstractmethod + def public_bytes(self, encoding, format): + """ + The serialized bytes of the public key. + """ + + +@six.add_metaclass(abc.ABCMeta) +class X448PrivateKey(object): + @classmethod + def generate(cls): + from cryptography.hazmat.backends.openssl.backend import backend + if not backend.x448_supported(): + raise UnsupportedAlgorithm( + "X448 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_EXCHANGE_ALGORITHM + ) + return backend.x448_generate_key() + + @classmethod + def from_private_bytes(cls, data): + from cryptography.hazmat.backends.openssl.backend import backend + if not backend.x448_supported(): + raise UnsupportedAlgorithm( + "X448 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_EXCHANGE_ALGORITHM + ) + + return backend.x448_load_private_bytes(data) + + @abc.abstractmethod + def public_key(self): + """ + The serialized bytes of the public key. + """ + + @abc.abstractmethod + def private_bytes(self, encoding, format, encryption_algorithm): + """ + The serialized bytes of the private key. + """ + + @abc.abstractmethod + def exchange(self, peer_public_key): + """ + Performs a key exchange operation using the provided peer's public key. + """ diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/ciphers/aead.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/ciphers/aead.py index 16899d0..42e19ad 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/ciphers/aead.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/ciphers/aead.py @@ -20,7 +20,7 @@ class ChaCha20Poly1305(object): "ChaCha20Poly1305 is not supported by this version of OpenSSL", exceptions._Reasons.UNSUPPORTED_CIPHER ) - utils._check_bytes("key", key) + utils._check_byteslike("key", key) if len(key) != 32: raise ValueError("ChaCha20Poly1305 key must be 32 bytes.") @@ -56,7 +56,7 @@ class ChaCha20Poly1305(object): ) def _check_params(self, nonce, data, associated_data): - utils._check_bytes("nonce", nonce) + utils._check_byteslike("nonce", nonce) utils._check_bytes("data", data) utils._check_bytes("associated_data", associated_data) if len(nonce) != 12: @@ -67,7 +67,7 @@ class AESCCM(object): _MAX_SIZE = 2 ** 32 def __init__(self, key, tag_length=16): - utils._check_bytes("key", key) + utils._check_byteslike("key", key) if len(key) not in (16, 24, 32): raise ValueError("AESCCM key must be 128, 192, or 256 bits.") @@ -129,7 +129,7 @@ class AESCCM(object): raise ValueError("Nonce too long for data") def _check_params(self, nonce, data, associated_data): - utils._check_bytes("nonce", nonce) + utils._check_byteslike("nonce", nonce) utils._check_bytes("data", data) utils._check_bytes("associated_data", associated_data) if not 7 <= len(nonce) <= 13: @@ -140,7 +140,7 @@ class AESGCM(object): _MAX_SIZE = 2 ** 32 def __init__(self, key): - utils._check_bytes("key", key) + utils._check_byteslike("key", key) if len(key) not in (16, 24, 32): raise ValueError("AESGCM key must be 128, 192, or 256 bits.") @@ -181,7 +181,7 @@ class AESGCM(object): ) def _check_params(self, nonce, data, associated_data): - utils._check_bytes("nonce", nonce) + utils._check_byteslike("nonce", nonce) utils._check_bytes("data", data) utils._check_bytes("associated_data", associated_data) if len(nonce) == 0: diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/ciphers/algorithms.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/ciphers/algorithms.py index 68a9e33..f4d5160 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/ciphers/algorithms.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/ciphers/algorithms.py @@ -13,11 +13,11 @@ from cryptography.hazmat.primitives.ciphers.modes import ModeWithNonce def _verify_key_size(algorithm, key): # Verify that the key is instance of bytes - utils._check_bytes("key", key) + utils._check_byteslike("key", key) # Verify that the key size matches the expected key size if len(key) * 8 not in algorithm.key_sizes: - raise ValueError("Invalid key size ({0}) for {1}.".format( + raise ValueError("Invalid key size ({}) for {}.".format( len(key) * 8, algorithm.name )) return key @@ -153,8 +153,7 @@ class ChaCha20(object): def __init__(self, key, nonce): self.key = _verify_key_size(self, key) - if not isinstance(nonce, bytes): - raise TypeError("nonce must be bytes") + utils._check_byteslike("nonce", nonce) if len(nonce) != 16: raise ValueError("nonce must be 128-bits (16 bytes)") diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/ciphers/base.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/ciphers/base.py index f857041..4d5f8d6 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/ciphers/base.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/ciphers/base.py @@ -179,7 +179,7 @@ class _AEADCipherContext(object): self._bytes_processed += data_size if self._bytes_processed > self._ctx._mode._MAX_ENCRYPTED_BYTES: raise ValueError( - "{0} has a maximum encrypted byte limit of {1}".format( + "{} has a maximum encrypted byte limit of {}".format( self._ctx._mode.name, self._ctx._mode._MAX_ENCRYPTED_BYTES ) ) @@ -217,7 +217,7 @@ class _AEADCipherContext(object): self._aad_bytes_processed += len(data) if self._aad_bytes_processed > self._ctx._mode._MAX_AAD_BYTES: raise ValueError( - "{0} has a maximum AAD byte limit of {1}".format( + "{} has a maximum AAD byte limit of {}".format( self._ctx._mode.name, self._ctx._mode._MAX_AAD_BYTES ) ) diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/ciphers/modes.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/ciphers/modes.py index e82c1a8..78fa1c4 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/ciphers/modes.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/ciphers/modes.py @@ -72,7 +72,7 @@ def _check_aes_key_length(self, algorithm): def _check_iv_length(self, algorithm): if len(self.initialization_vector) * 8 != algorithm.block_size: - raise ValueError("Invalid IV size ({0}) for {1}.".format( + raise ValueError("Invalid IV size ({}) for {}.".format( len(self.initialization_vector), self.name )) @@ -88,9 +88,7 @@ class CBC(object): name = "CBC" def __init__(self, initialization_vector): - if not isinstance(initialization_vector, bytes): - raise TypeError("initialization_vector must be bytes") - + utils._check_byteslike("initialization_vector", initialization_vector) self._initialization_vector = initialization_vector initialization_vector = utils.read_only_property("_initialization_vector") @@ -103,8 +101,7 @@ class XTS(object): name = "XTS" def __init__(self, tweak): - if not isinstance(tweak, bytes): - raise TypeError("tweak must be bytes") + utils._check_byteslike("tweak", tweak) if len(tweak) != 16: raise ValueError("tweak must be 128-bits (16 bytes)") @@ -134,9 +131,7 @@ class OFB(object): name = "OFB" def __init__(self, initialization_vector): - if not isinstance(initialization_vector, bytes): - raise TypeError("initialization_vector must be bytes") - + utils._check_byteslike("initialization_vector", initialization_vector) self._initialization_vector = initialization_vector initialization_vector = utils.read_only_property("_initialization_vector") @@ -149,9 +144,7 @@ class CFB(object): name = "CFB" def __init__(self, initialization_vector): - if not isinstance(initialization_vector, bytes): - raise TypeError("initialization_vector must be bytes") - + utils._check_byteslike("initialization_vector", initialization_vector) self._initialization_vector = initialization_vector initialization_vector = utils.read_only_property("_initialization_vector") @@ -164,9 +157,7 @@ class CFB8(object): name = "CFB8" def __init__(self, initialization_vector): - if not isinstance(initialization_vector, bytes): - raise TypeError("initialization_vector must be bytes") - + utils._check_byteslike("initialization_vector", initialization_vector) self._initialization_vector = initialization_vector initialization_vector = utils.read_only_property("_initialization_vector") @@ -179,9 +170,7 @@ class CTR(object): name = "CTR" def __init__(self, nonce): - if not isinstance(nonce, bytes): - raise TypeError("nonce must be bytes") - + utils._check_byteslike("nonce", nonce) self._nonce = nonce nonce = utils.read_only_property("_nonce") @@ -189,7 +178,7 @@ class CTR(object): def validate_for_algorithm(self, algorithm): _check_aes_key_length(self, algorithm) if len(self.nonce) * 8 != algorithm.block_size: - raise ValueError("Invalid nonce size ({0}) for {1}.".format( + raise ValueError("Invalid nonce size ({}) for {}.".format( len(self.nonce), self.name )) @@ -206,19 +195,17 @@ class GCM(object): # len(initialization_vector) must in [1, 2 ** 64), but it's impossible # to actually construct a bytes object that large, so we don't check # for it - if not isinstance(initialization_vector, bytes): - raise TypeError("initialization_vector must be bytes") + utils._check_byteslike("initialization_vector", initialization_vector) if len(initialization_vector) == 0: raise ValueError("initialization_vector must be at least 1 byte") self._initialization_vector = initialization_vector if tag is not None: - if not isinstance(tag, bytes): - raise TypeError("tag must be bytes or None") + utils._check_bytes("tag", tag) if min_tag_length < 4: raise ValueError("min_tag_length must be >= 4") if len(tag) < min_tag_length: raise ValueError( - "Authentication tag must be {0} bytes or longer.".format( + "Authentication tag must be {} bytes or longer.".format( min_tag_length) ) self._tag = tag diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/cmac.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/cmac.py index 77537f0..95a8d97 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/cmac.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/cmac.py @@ -9,10 +9,9 @@ from cryptography.exceptions import ( AlreadyFinalized, UnsupportedAlgorithm, _Reasons ) from cryptography.hazmat.backends.interfaces import CMACBackend -from cryptography.hazmat.primitives import ciphers, mac +from cryptography.hazmat.primitives import ciphers -@utils.register_interface(mac.MACContext) class CMAC(object): def __init__(self, algorithm, backend, ctx=None): if not isinstance(backend, CMACBackend): @@ -36,8 +35,8 @@ class CMAC(object): def update(self, data): if self._ctx is None: raise AlreadyFinalized("Context was already finalized.") - if not isinstance(data, bytes): - raise TypeError("data must be bytes.") + + utils._check_bytes("data", data) self._ctx.update(data) def finalize(self): @@ -48,8 +47,7 @@ class CMAC(object): return digest def verify(self, signature): - if not isinstance(signature, bytes): - raise TypeError("signature must be bytes.") + utils._check_bytes("signature", signature) if self._ctx is None: raise AlreadyFinalized("Context was already finalized.") diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/constant_time.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/constant_time.py index 0e987ea..35ceafe 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/constant_time.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/constant_time.py @@ -21,9 +21,9 @@ if hasattr(hmac, "compare_digest"): else: warnings.warn( "Support for your Python version is deprecated. The next version of " - "cryptography will remove support. Please upgrade to a 2.7.x " - "release that supports hmac.compare_digest as soon as possible.", - utils.DeprecatedIn23, + "cryptography will remove support. Please upgrade to a release " + "(2.7.7+) that supports hmac.compare_digest as soon as possible.", + utils.PersistentlyDeprecated2018, ) def bytes_eq(a, b): diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/hashes.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/hashes.py index 3f3aadd..9be2b60 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/hashes.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/hashes.py @@ -57,6 +57,13 @@ class HashContext(object): """ +@six.add_metaclass(abc.ABCMeta) +class ExtendableOutputFunction(object): + """ + An interface for extendable output functions. + """ + + @utils.register_interface(HashContext) class Hash(object): def __init__(self, algorithm, backend, ctx=None): @@ -82,8 +89,7 @@ class Hash(object): def update(self, data): if self._ctx is None: raise AlreadyFinalized("Context was already finalized.") - if not isinstance(data, bytes): - raise TypeError("data must be bytes.") + utils._check_byteslike("data", data) self._ctx.update(data) def copy(self): @@ -108,6 +114,20 @@ class SHA1(object): block_size = 64 +@utils.register_interface(HashAlgorithm) +class SHA512_224(object): # noqa: N801 + name = "sha512-224" + digest_size = 28 + block_size = 128 + + +@utils.register_interface(HashAlgorithm) +class SHA512_256(object): # noqa: N801 + name = "sha512-256" + digest_size = 32 + block_size = 128 + + @utils.register_interface(HashAlgorithm) class SHA224(object): name = "sha224" @@ -136,6 +156,64 @@ class SHA512(object): block_size = 128 +@utils.register_interface(HashAlgorithm) +class SHA3_224(object): # noqa: N801 + name = "sha3-224" + digest_size = 28 + + +@utils.register_interface(HashAlgorithm) +class SHA3_256(object): # noqa: N801 + name = "sha3-256" + digest_size = 32 + + +@utils.register_interface(HashAlgorithm) +class SHA3_384(object): # noqa: N801 + name = "sha3-384" + digest_size = 48 + + +@utils.register_interface(HashAlgorithm) +class SHA3_512(object): # noqa: N801 + name = "sha3-512" + digest_size = 64 + + +@utils.register_interface(HashAlgorithm) +@utils.register_interface(ExtendableOutputFunction) +class SHAKE128(object): + name = "shake128" + + def __init__(self, digest_size): + if not isinstance(digest_size, six.integer_types): + raise TypeError("digest_size must be an integer") + + if digest_size < 1: + raise ValueError("digest_size must be a positive integer") + + self._digest_size = digest_size + + digest_size = utils.read_only_property("_digest_size") + + +@utils.register_interface(HashAlgorithm) +@utils.register_interface(ExtendableOutputFunction) +class SHAKE256(object): + name = "shake256" + + def __init__(self, digest_size): + if not isinstance(digest_size, six.integer_types): + raise TypeError("digest_size must be an integer") + + if digest_size < 1: + raise ValueError("digest_size must be a positive integer") + + self._digest_size = digest_size + + digest_size = utils.read_only_property("_digest_size") + + @utils.register_interface(HashAlgorithm) class MD5(object): name = "md5" @@ -151,13 +229,9 @@ class BLAKE2b(object): block_size = 128 def __init__(self, digest_size): - if ( - digest_size > self._max_digest_size or - digest_size < self._min_digest_size - ): - raise ValueError("Digest size must be {0}-{1}".format( - self._min_digest_size, self._max_digest_size) - ) + + if digest_size != 64: + raise ValueError("Digest size must be 64") self._digest_size = digest_size @@ -172,13 +246,9 @@ class BLAKE2s(object): _min_digest_size = 1 def __init__(self, digest_size): - if ( - digest_size > self._max_digest_size or - digest_size < self._min_digest_size - ): - raise ValueError("Digest size must be {0}-{1}".format( - self._min_digest_size, self._max_digest_size) - ) + + if digest_size != 32: + raise ValueError("Digest size must be 32") self._digest_size = digest_size diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/hmac.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/hmac.py index 2e9a4e2..9eceeac 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/hmac.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/hmac.py @@ -9,10 +9,9 @@ from cryptography.exceptions import ( AlreadyFinalized, UnsupportedAlgorithm, _Reasons ) from cryptography.hazmat.backends.interfaces import HMACBackend -from cryptography.hazmat.primitives import hashes, mac +from cryptography.hazmat.primitives import hashes -@utils.register_interface(mac.MACContext) @utils.register_interface(hashes.HashContext) class HMAC(object): def __init__(self, key, algorithm, backend, ctx=None): @@ -38,8 +37,7 @@ class HMAC(object): def update(self, data): if self._ctx is None: raise AlreadyFinalized("Context was already finalized.") - if not isinstance(data, bytes): - raise TypeError("data must be bytes.") + utils._check_byteslike("data", data) self._ctx.update(data) def copy(self): @@ -60,8 +58,7 @@ class HMAC(object): return digest def verify(self, signature): - if not isinstance(signature, bytes): - raise TypeError("signature must be bytes.") + utils._check_bytes("signature", signature) if self._ctx is None: raise AlreadyFinalized("Context was already finalized.") diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/kdf/concatkdf.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/kdf/concatkdf.py index c6399e4..7cb6385 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/kdf/concatkdf.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/kdf/concatkdf.py @@ -24,17 +24,15 @@ def _common_args_checks(algorithm, length, otherinfo): max_length = algorithm.digest_size * (2 ** 32 - 1) if length > max_length: raise ValueError( - "Can not derive keys larger than {0} bits.".format( + "Can not derive keys larger than {} bits.".format( max_length )) - if not (otherinfo is None or isinstance(otherinfo, bytes)): - raise TypeError("otherinfo must be bytes.") + if otherinfo is not None: + utils._check_bytes("otherinfo", otherinfo) def _concatkdf_derive(key_material, length, auxfn, otherinfo): - if not isinstance(key_material, bytes): - raise TypeError("key_material must be bytes.") - + utils._check_byteslike("key_material", key_material) output = [b""] outlen = 0 counter = 1 @@ -96,10 +94,11 @@ class ConcatKDFHMAC(object): if self._otherinfo is None: self._otherinfo = b"" - if not (salt is None or isinstance(salt, bytes)): - raise TypeError("salt must be bytes.") if salt is None: salt = b"\x00" * algorithm.block_size + else: + utils._check_bytes("salt", salt) + self._salt = salt if not isinstance(backend, HMACBackend): diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/kdf/hkdf.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/kdf/hkdf.py index 917b4e9..01f0f28 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/kdf/hkdf.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/kdf/hkdf.py @@ -26,11 +26,10 @@ class HKDF(object): self._algorithm = algorithm - if not (salt is None or isinstance(salt, bytes)): - raise TypeError("salt must be bytes.") - if salt is None: salt = b"\x00" * self._algorithm.digest_size + else: + utils._check_bytes("salt", salt) self._salt = salt @@ -44,9 +43,7 @@ class HKDF(object): return h.finalize() def derive(self, key_material): - if not isinstance(key_material, bytes): - raise TypeError("key_material must be bytes.") - + utils._check_byteslike("key_material", key_material) return self._hkdf_expand.derive(self._extract(key_material)) def verify(self, key_material, expected_key): @@ -71,17 +68,16 @@ class HKDFExpand(object): if length > max_length: raise ValueError( - "Can not derive keys larger than {0} octets.".format( + "Can not derive keys larger than {} octets.".format( max_length )) self._length = length - if not (info is None or isinstance(info, bytes)): - raise TypeError("info must be bytes.") - if info is None: info = b"" + else: + utils._check_bytes("info", info) self._info = info @@ -102,9 +98,7 @@ class HKDFExpand(object): return b"".join(output)[:self._length] def derive(self, key_material): - if not isinstance(key_material, bytes): - raise TypeError("key_material must be bytes.") - + utils._check_byteslike("key_material", key_material) if self._used: raise AlreadyFinalized diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/kdf/kbkdf.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/kdf/kbkdf.py index 14de56e..56783a8 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/kdf/kbkdf.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/kdf/kbkdf.py @@ -73,10 +73,8 @@ class KBKDFHMAC(object): if context is None: context = b'' - if (not isinstance(label, bytes) or - not isinstance(context, bytes)): - raise TypeError('label and context must be of type bytes') - + utils._check_bytes("label", label) + utils._check_bytes("context", context) self._algorithm = algorithm self._mode = mode self._length = length @@ -102,8 +100,7 @@ class KBKDFHMAC(object): if self._used: raise AlreadyFinalized - if not isinstance(key_material, bytes): - raise TypeError('key_material must be bytes') + utils._check_byteslike("key_material", key_material) self._used = True # inverse floor division (equivalent to ceiling) diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/kdf/pbkdf2.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/kdf/pbkdf2.py index f8ce7a3..07d8ac6 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/kdf/pbkdf2.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/kdf/pbkdf2.py @@ -24,15 +24,14 @@ class PBKDF2HMAC(object): if not backend.pbkdf2_hmac_supported(algorithm): raise UnsupportedAlgorithm( - "{0} is not supported for PBKDF2 by this backend.".format( + "{} is not supported for PBKDF2 by this backend.".format( algorithm.name), _Reasons.UNSUPPORTED_HASH ) self._used = False self._algorithm = algorithm self._length = length - if not isinstance(salt, bytes): - raise TypeError("salt must be bytes.") + utils._check_bytes("salt", salt) self._salt = salt self._iterations = iterations self._backend = backend @@ -42,8 +41,7 @@ class PBKDF2HMAC(object): raise AlreadyFinalized("PBKDF2 instances can only be used once.") self._used = True - if not isinstance(key_material, bytes): - raise TypeError("key_material must be bytes.") + utils._check_byteslike("key_material", key_material) return self._backend.derive_pbkdf2_hmac( self._algorithm, self._length, diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/kdf/scrypt.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/kdf/scrypt.py index 77dcf9a..df9745e 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/kdf/scrypt.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/kdf/scrypt.py @@ -30,9 +30,7 @@ class Scrypt(object): ) self._length = length - if not isinstance(salt, bytes): - raise TypeError("salt must be bytes.") - + utils._check_bytes("salt", salt) if n < 2 or (n & (n - 1)) != 0: raise ValueError("n must be greater than 1 and be a power of 2.") @@ -54,8 +52,7 @@ class Scrypt(object): raise AlreadyFinalized("Scrypt instances can only be used once.") self._used = True - if not isinstance(key_material, bytes): - raise TypeError("key_material must be bytes.") + utils._check_byteslike("key_material", key_material) return self._backend.derive_scrypt( key_material, self._salt, self._length, self._n, self._r, self._p ) diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/kdf/x963kdf.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/kdf/x963kdf.py index 83789b3..9eb50b0 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/kdf/x963kdf.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/kdf/x963kdf.py @@ -26,9 +26,10 @@ class X963KDF(object): max_len = algorithm.digest_size * (2 ** 32 - 1) if length > max_len: raise ValueError( - "Can not derive keys larger than {0} bits.".format(max_len)) - if not (sharedinfo is None or isinstance(sharedinfo, bytes)): - raise TypeError("sharedinfo must be bytes.") + "Can not derive keys larger than {} bits.".format(max_len)) + if sharedinfo is not None: + utils._check_bytes("sharedinfo", sharedinfo) + self._algorithm = algorithm self._length = length self._sharedinfo = sharedinfo @@ -45,10 +46,7 @@ class X963KDF(object): if self._used: raise AlreadyFinalized self._used = True - - if not isinstance(key_material, bytes): - raise TypeError("key_material must be bytes.") - + utils._check_byteslike("key_material", key_material) output = [b""] outlen = 0 counter = 1 diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/mac.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/mac.py deleted file mode 100644 index 4c95190..0000000 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/mac.py +++ /dev/null @@ -1,37 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - -from __future__ import absolute_import, division, print_function - -import abc - -import six - - -@six.add_metaclass(abc.ABCMeta) -class MACContext(object): - @abc.abstractmethod - def update(self, data): - """ - Processes the provided bytes. - """ - - @abc.abstractmethod - def finalize(self): - """ - Returns the message authentication code as bytes. - """ - - @abc.abstractmethod - def copy(self): - """ - Return a MACContext that is a copy of the current context. - """ - - @abc.abstractmethod - def verify(self, signature): - """ - Checks if the generated message authentication code matches the - signature. - """ diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/padding.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/padding.py index a081976..170c802 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/padding.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/padding.py @@ -40,8 +40,7 @@ def _byte_padding_update(buffer_, data, block_size): if buffer_ is None: raise AlreadyFinalized("Context was already finalized.") - if not isinstance(data, bytes): - raise TypeError("data must be bytes.") + utils._check_bytes("data", data) buffer_ += data @@ -65,8 +64,7 @@ def _byte_unpadding_update(buffer_, data, block_size): if buffer_ is None: raise AlreadyFinalized("Context was already finalized.") - if not isinstance(data, bytes): - raise TypeError("data must be bytes.") + utils._check_bytes("data", data) buffer_ += data diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/poly1305.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/poly1305.py new file mode 100644 index 0000000..d92f62a --- /dev/null +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/poly1305.py @@ -0,0 +1,55 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + + +from cryptography import utils +from cryptography.exceptions import ( + AlreadyFinalized, UnsupportedAlgorithm, _Reasons +) + + +class Poly1305(object): + def __init__(self, key): + from cryptography.hazmat.backends.openssl.backend import backend + if not backend.poly1305_supported(): + raise UnsupportedAlgorithm( + "poly1305 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_MAC + ) + self._ctx = backend.create_poly1305_ctx(key) + + def update(self, data): + if self._ctx is None: + raise AlreadyFinalized("Context was already finalized.") + utils._check_byteslike("data", data) + self._ctx.update(data) + + def finalize(self): + if self._ctx is None: + raise AlreadyFinalized("Context was already finalized.") + mac = self._ctx.finalize() + self._ctx = None + return mac + + def verify(self, tag): + utils._check_bytes("tag", tag) + if self._ctx is None: + raise AlreadyFinalized("Context was already finalized.") + + ctx, self._ctx = self._ctx, None + ctx.verify(tag) + + @classmethod + def generate_tag(cls, key, data): + p = Poly1305(key) + p.update(data) + return p.finalize() + + @classmethod + def verify_tag(cls, key, data, tag): + p = Poly1305(key) + p.update(data) + p.verify(tag) diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/serialization/__init__.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/serialization/__init__.py new file mode 100644 index 0000000..f6d4ce9 --- /dev/null +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/serialization/__init__.py @@ -0,0 +1,26 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from cryptography.hazmat.primitives.serialization.base import ( + BestAvailableEncryption, Encoding, KeySerializationEncryption, + NoEncryption, ParameterFormat, PrivateFormat, PublicFormat, + load_der_parameters, load_der_private_key, load_der_public_key, + load_pem_parameters, load_pem_private_key, load_pem_public_key, +) +from cryptography.hazmat.primitives.serialization.ssh import ( + load_ssh_public_key +) + + +_PEM_DER = (Encoding.PEM, Encoding.DER) + +__all__ = [ + "load_der_parameters", "load_der_private_key", "load_der_public_key", + "load_pem_parameters", "load_pem_private_key", "load_pem_public_key", + "load_ssh_public_key", "Encoding", "PrivateFormat", "PublicFormat", + "ParameterFormat", "KeySerializationEncryption", "BestAvailableEncryption", + "NoEncryption", +] diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/serialization/base.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/serialization/base.py new file mode 100644 index 0000000..4218ea8 --- /dev/null +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/serialization/base.py @@ -0,0 +1,82 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import abc +from enum import Enum + +import six + +from cryptography import utils + + +def load_pem_private_key(data, password, backend): + return backend.load_pem_private_key(data, password) + + +def load_pem_public_key(data, backend): + return backend.load_pem_public_key(data) + + +def load_pem_parameters(data, backend): + return backend.load_pem_parameters(data) + + +def load_der_private_key(data, password, backend): + return backend.load_der_private_key(data, password) + + +def load_der_public_key(data, backend): + return backend.load_der_public_key(data) + + +def load_der_parameters(data, backend): + return backend.load_der_parameters(data) + + +class Encoding(Enum): + PEM = "PEM" + DER = "DER" + OpenSSH = "OpenSSH" + Raw = "Raw" + X962 = "ANSI X9.62" + + +class PrivateFormat(Enum): + PKCS8 = "PKCS8" + TraditionalOpenSSL = "TraditionalOpenSSL" + Raw = "Raw" + + +class PublicFormat(Enum): + SubjectPublicKeyInfo = "X.509 subjectPublicKeyInfo with PKCS#1" + PKCS1 = "Raw PKCS#1" + OpenSSH = "OpenSSH" + Raw = "Raw" + CompressedPoint = "X9.62 Compressed Point" + UncompressedPoint = "X9.62 Uncompressed Point" + + +class ParameterFormat(Enum): + PKCS3 = "PKCS3" + + +@six.add_metaclass(abc.ABCMeta) +class KeySerializationEncryption(object): + pass + + +@utils.register_interface(KeySerializationEncryption) +class BestAvailableEncryption(object): + def __init__(self, password): + if not isinstance(password, bytes) or len(password) == 0: + raise ValueError("Password must be 1 or more bytes.") + + self.password = password + + +@utils.register_interface(KeySerializationEncryption) +class NoEncryption(object): + pass diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/serialization/pkcs12.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/serialization/pkcs12.py new file mode 100644 index 0000000..98161d5 --- /dev/null +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/serialization/pkcs12.py @@ -0,0 +1,9 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + + +def load_key_and_certificates(data, password, backend): + return backend.load_key_and_certificates_from_pkcs12(data, password) diff --git a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/serialization.py b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/serialization/ssh.py similarity index 70% rename from server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/serialization.py rename to server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/serialization/ssh.py index bd09e6e..a1d6c8c 100644 --- a/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/serialization.py +++ b/server/www/packages/packages-linux/x64/cryptography/hazmat/primitives/serialization/ssh.py @@ -4,40 +4,14 @@ from __future__ import absolute_import, division, print_function -import abc import base64 import struct -from enum import Enum import six from cryptography import utils from cryptography.exceptions import UnsupportedAlgorithm -from cryptography.hazmat.primitives.asymmetric import dsa, ec, rsa - - -def load_pem_private_key(data, password, backend): - return backend.load_pem_private_key(data, password) - - -def load_pem_public_key(data, backend): - return backend.load_pem_public_key(data) - - -def load_pem_parameters(data, backend): - return backend.load_pem_parameters(data) - - -def load_der_private_key(data, password, backend): - return backend.load_der_private_key(data, password) - - -def load_der_public_key(data, backend): - return backend.load_der_public_key(data) - - -def load_der_parameters(data, backend): - return backend.load_der_parameters(data) +from cryptography.hazmat.primitives.asymmetric import dsa, ec, ed25519, rsa def load_ssh_public_key(data, backend): @@ -57,6 +31,8 @@ def load_ssh_public_key(data, backend): b'ecdsa-sha2-nistp256', b'ecdsa-sha2-nistp384', b'ecdsa-sha2-nistp521', ]: loader = _load_ssh_ecdsa_public_key + elif key_type == b'ssh-ed25519': + loader = _load_ssh_ed25519_public_key else: raise UnsupportedAlgorithm('Key type is not supported.') @@ -125,8 +101,16 @@ def _load_ssh_ecdsa_public_key(expected_key_type, decoded_data, backend): "Compressed elliptic curve points are not supported" ) - numbers = ec.EllipticCurvePublicNumbers.from_encoded_point(curve, data) - return numbers.public_key(backend) + return ec.EllipticCurvePublicKey.from_encoded_point(curve, data) + + +def _load_ssh_ed25519_public_key(expected_key_type, decoded_data, backend): + data, rest = _ssh_read_next_string(decoded_data) + + if rest: + raise ValueError('Key body contains extra bytes.') + + return ed25519.Ed25519PublicKey.from_public_bytes(data) def _ssh_read_next_string(data): @@ -167,43 +151,3 @@ def _ssh_write_mpint(value): if six.indexbytes(data, 0) & 0x80: data = b"\x00" + data return _ssh_write_string(data) - - -class Encoding(Enum): - PEM = "PEM" - DER = "DER" - OpenSSH = "OpenSSH" - - -class PrivateFormat(Enum): - PKCS8 = "PKCS8" - TraditionalOpenSSL = "TraditionalOpenSSL" - - -class PublicFormat(Enum): - SubjectPublicKeyInfo = "X.509 subjectPublicKeyInfo with PKCS#1" - PKCS1 = "Raw PKCS#1" - OpenSSH = "OpenSSH" - - -class ParameterFormat(Enum): - PKCS3 = "PKCS3" - - -@six.add_metaclass(abc.ABCMeta) -class KeySerializationEncryption(object): - pass - - -@utils.register_interface(KeySerializationEncryption) -class BestAvailableEncryption(object): - def __init__(self, password): - if not isinstance(password, bytes) or len(password) == 0: - raise ValueError("Password must be 1 or more bytes.") - - self.password = password - - -@utils.register_interface(KeySerializationEncryption) -class NoEncryption(object): - pass diff --git a/server/www/packages/packages-linux/x64/cryptography/utils.py b/server/www/packages/packages-linux/x64/cryptography/utils.py index 3d45a77..0b36f63 100644 --- a/server/www/packages/packages-linux/x64/cryptography/utils.py +++ b/server/www/packages/packages-linux/x64/cryptography/utils.py @@ -20,14 +20,22 @@ class CryptographyDeprecationWarning(UserWarning): # Several APIs were deprecated with no specific end-of-life date because of the # ubiquity of their use. They should not be removed until we agree on when that # cycle ends. -PersistentlyDeprecated = CryptographyDeprecationWarning -DeprecatedIn21 = CryptographyDeprecationWarning -DeprecatedIn23 = CryptographyDeprecationWarning +PersistentlyDeprecated2017 = CryptographyDeprecationWarning +PersistentlyDeprecated2018 = CryptographyDeprecationWarning +DeprecatedIn25 = CryptographyDeprecationWarning +DeprecatedIn27 = CryptographyDeprecationWarning def _check_bytes(name, value): if not isinstance(value, bytes): - raise TypeError("{0} must be bytes".format(name)) + raise TypeError("{} must be bytes".format(name)) + + +def _check_byteslike(name, value): + try: + memoryview(value) + except TypeError: + raise TypeError("{} must be bytes-like".format(name)) def read_only_property(name): @@ -90,7 +98,7 @@ def verify_interface(iface, klass): for method in iface.__abstractmethods__: if not hasattr(klass, method): raise InterfaceNotImplemented( - "{0} is missing a {1!r} method".format(klass, method) + "{} is missing a {!r} method".format(klass, method) ) if isinstance(getattr(iface, method), abc.abstractproperty): # Can't properly verify these yet. @@ -99,8 +107,8 @@ def verify_interface(iface, klass): actual = signature(getattr(klass, method)) if sig != actual: raise InterfaceNotImplemented( - "{0}.{1}'s signature differs from the expected. Expected: " - "{2!r}. Received: {3!r}".format( + "{}.{}'s signature differs from the expected. Expected: " + "{!r}. Received: {!r}".format( klass, method, sig, actual ) ) @@ -152,7 +160,7 @@ def deprecated(value, module_name, message, warning_class): def cached_property(func): - cached_name = "_cached_{0}".format(func) + cached_name = "_cached_{}".format(func) sentinel = object() def inner(instance): diff --git a/server/www/packages/packages-linux/x64/cryptography/x509/__init__.py b/server/www/packages/packages-linux/x64/cryptography/x509/__init__.py index fd01945..b761e26 100644 --- a/server/www/packages/packages-linux/x64/cryptography/x509/__init__.py +++ b/server/www/packages/packages-linux/x64/cryptography/x509/__init__.py @@ -21,9 +21,9 @@ from cryptography.x509.extensions import ( DeltaCRLIndicator, DistributionPoint, DuplicateExtension, ExtendedKeyUsage, Extension, ExtensionNotFound, ExtensionType, Extensions, FreshestCRL, GeneralNames, InhibitAnyPolicy, InvalidityDate, IssuerAlternativeName, - KeyUsage, NameConstraints, NoticeReference, OCSPNoCheck, OCSPNonce, - PolicyConstraints, PolicyInformation, PrecertPoison, - PrecertificateSignedCertificateTimestamps, ReasonFlags, + IssuingDistributionPoint, KeyUsage, NameConstraints, NoticeReference, + OCSPNoCheck, OCSPNonce, PolicyConstraints, PolicyInformation, + PrecertPoison, PrecertificateSignedCertificateTimestamps, ReasonFlags, SubjectAlternativeName, SubjectKeyIdentifier, TLSFeature, TLSFeatureType, UnrecognizedExtension, UserNotice ) @@ -134,6 +134,7 @@ __all__ = [ "Extension", "ExtendedKeyUsage", "FreshestCRL", + "IssuingDistributionPoint", "TLSFeature", "TLSFeatureType", "OCSPNoCheck", diff --git a/server/www/packages/packages-linux/x64/cryptography/x509/base.py b/server/www/packages/packages-linux/x64/cryptography/x509/base.py index a3b334a..3983c9b 100644 --- a/server/www/packages/packages-linux/x64/cryptography/x509/base.py +++ b/server/www/packages/packages-linux/x64/cryptography/x509/base.py @@ -12,12 +12,14 @@ from enum import Enum import six from cryptography import utils -from cryptography.hazmat.primitives.asymmetric import dsa, ec, rsa +from cryptography.hazmat.primitives.asymmetric import ( + dsa, ec, ed25519, ed448, rsa +) from cryptography.x509.extensions import Extension, ExtensionType from cryptography.x509.name import Name -_UNIX_EPOCH = datetime.datetime(1970, 1, 1) +_EARLIEST_UTC_TIME = datetime.datetime(1950, 1, 1) def _reject_duplicate_extension(extension, extensions): @@ -474,9 +476,12 @@ class CertificateBuilder(object): Sets the requestor's public key (as found in the signing request). """ if not isinstance(key, (dsa.DSAPublicKey, rsa.RSAPublicKey, - ec.EllipticCurvePublicKey)): + ec.EllipticCurvePublicKey, + ed25519.Ed25519PublicKey, + ed448.Ed448PublicKey)): raise TypeError('Expecting one of DSAPublicKey, RSAPublicKey,' - ' or EllipticCurvePublicKey.') + ' EllipticCurvePublicKey, Ed25519PublicKey or' + ' Ed448PublicKey.') if self._public_key is not None: raise ValueError('The public key may only be set once.') return CertificateBuilder( @@ -516,9 +521,9 @@ class CertificateBuilder(object): if self._not_valid_before is not None: raise ValueError('The not valid before may only be set once.') time = _convert_to_naive_utc_time(time) - if time <= _UNIX_EPOCH: - raise ValueError('The not valid before date must be after the unix' - ' epoch (1970 January 1).') + if time < _EARLIEST_UTC_TIME: + raise ValueError('The not valid before date must be on or after' + ' 1950 January 1).') if self._not_valid_after is not None and time > self._not_valid_after: raise ValueError( 'The not valid before date must be before the not valid after ' @@ -539,9 +544,9 @@ class CertificateBuilder(object): if self._not_valid_after is not None: raise ValueError('The not valid after may only be set once.') time = _convert_to_naive_utc_time(time) - if time <= _UNIX_EPOCH: - raise ValueError('The not valid after date must be after the unix' - ' epoch (1970 January 1).') + if time < _EARLIEST_UTC_TIME: + raise ValueError('The not valid after date must be on or after' + ' 1950 January 1.') if (self._not_valid_before is not None and time < self._not_valid_before): raise ValueError( @@ -620,9 +625,9 @@ class CertificateRevocationListBuilder(object): if self._last_update is not None: raise ValueError('Last update may only be set once.') last_update = _convert_to_naive_utc_time(last_update) - if last_update <= _UNIX_EPOCH: - raise ValueError('The last update date must be after the unix' - ' epoch (1970 January 1).') + if last_update < _EARLIEST_UTC_TIME: + raise ValueError('The last update date must be on or after' + ' 1950 January 1.') if self._next_update is not None and last_update > self._next_update: raise ValueError( 'The last update date must be before the next update date.' @@ -638,9 +643,9 @@ class CertificateRevocationListBuilder(object): if self._next_update is not None: raise ValueError('Last update may only be set once.') next_update = _convert_to_naive_utc_time(next_update) - if next_update <= _UNIX_EPOCH: - raise ValueError('The last update date must be after the unix' - ' epoch (1970 January 1).') + if next_update < _EARLIEST_UTC_TIME: + raise ValueError('The last update date must be on or after' + ' 1950 January 1.') if self._last_update is not None and next_update < self._last_update: raise ValueError( 'The next update date must be after the last update date.' @@ -720,9 +725,9 @@ class RevokedCertificateBuilder(object): if self._revocation_date is not None: raise ValueError('The revocation date may only be set once.') time = _convert_to_naive_utc_time(time) - if time <= _UNIX_EPOCH: - raise ValueError('The revocation date must be after the unix' - ' epoch (1970 January 1).') + if time < _EARLIEST_UTC_TIME: + raise ValueError('The revocation date must be on or after' + ' 1950 January 1.') return RevokedCertificateBuilder( self._serial_number, time, self._extensions ) diff --git a/server/www/packages/packages-linux/x64/cryptography/x509/extensions.py b/server/www/packages/packages-linux/x64/cryptography/x509/extensions.py index 6301af5..f60075a 100644 --- a/server/www/packages/packages-linux/x64/cryptography/x509/extensions.py +++ b/server/www/packages/packages-linux/x64/cryptography/x509/extensions.py @@ -8,13 +8,15 @@ import abc import datetime import hashlib import ipaddress +import warnings from enum import Enum -from asn1crypto.keys import PublicKeyInfo - import six from cryptography import utils +from cryptography.hazmat._der import ( + BIT_STRING, DERReader, OBJECT_IDENTIFIER, SEQUENCE +) from cryptography.hazmat.primitives import constant_time, serialization from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePublicKey from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey @@ -35,7 +37,10 @@ def _key_identifier_from_public_key(public_key): serialization.PublicFormat.PKCS1, ) elif isinstance(public_key, EllipticCurvePublicKey): - data = public_key.public_numbers().encode_point() + data = public_key.public_bytes( + serialization.Encoding.X962, + serialization.PublicFormat.UncompressedPoint + ) else: # This is a very slow way to do this. serialized = public_key.public_bytes( @@ -43,11 +48,41 @@ def _key_identifier_from_public_key(public_key): serialization.PublicFormat.SubjectPublicKeyInfo ) - data = six.binary_type(PublicKeyInfo.load(serialized)['public_key']) + reader = DERReader(serialized) + with reader.read_single_element(SEQUENCE) as public_key_info: + algorithm = public_key_info.read_element(SEQUENCE) + public_key = public_key_info.read_element(BIT_STRING) + + # Double-check the algorithm structure. + with algorithm: + algorithm.read_element(OBJECT_IDENTIFIER) + if not algorithm.is_empty(): + # Skip the optional parameters field. + algorithm.read_any_element() + + # BIT STRING contents begin with the number of padding bytes added. It + # must be zero for SubjectPublicKeyInfo structures. + if public_key.read_byte() != 0: + raise ValueError('Invalid public key encoding') + + data = public_key.data return hashlib.sha1(data).digest() +def _make_sequence_methods(field_name): + def len_method(self): + return len(getattr(self, field_name)) + + def iter_method(self): + return iter(getattr(self, field_name)) + + def getitem_method(self, idx): + return getattr(self, field_name)[idx] + + return len_method, iter_method, getitem_method + + class DuplicateExtension(Exception): def __init__(self, msg, oid): super(DuplicateExtension, self).__init__(msg) @@ -78,7 +113,7 @@ class Extensions(object): if ext.oid == oid: return ext - raise ExtensionNotFound("No {0} extension was found".format(oid), oid) + raise ExtensionNotFound("No {} extension was found".format(oid), oid) def get_extension_for_class(self, extclass): if extclass is UnrecognizedExtension: @@ -93,21 +128,14 @@ class Extensions(object): return ext raise ExtensionNotFound( - "No {0} extension was found".format(extclass), extclass.oid + "No {} extension was found".format(extclass), extclass.oid ) - def __iter__(self): - return iter(self._extensions) - - def __len__(self): - return len(self._extensions) - - def __getitem__(self, idx): - return self._extensions[idx] + __len__, __iter__, __getitem__ = _make_sequence_methods("_extensions") def __repr__(self): return ( - "".format(self._extensions) + "".format(self._extensions) ) @@ -134,7 +162,7 @@ class CRLNumber(object): return hash(self.crl_number) def __repr__(self): - return "".format(self.crl_number) + return "".format(self.crl_number) crl_number = utils.read_only_property("_crl_number") @@ -185,8 +213,21 @@ class AuthorityKeyIdentifier(object): @classmethod def from_issuer_subject_key_identifier(cls, ski): + if isinstance(ski, SubjectKeyIdentifier): + digest = ski.digest + else: + digest = ski.value.digest + warnings.warn( + "Extension objects are deprecated as arguments to " + "from_issuer_subject_key_identifier and support will be " + "removed soon. Please migrate to passing a " + "SubjectKeyIdentifier directly.", + utils.DeprecatedIn27, + stacklevel=2, + ) + return cls( - key_identifier=ski.value.digest, + key_identifier=digest, authority_cert_issuer=None, authority_cert_serial_number=None ) @@ -272,14 +313,10 @@ class AuthorityInformationAccess(object): self._descriptions = descriptions - def __iter__(self): - return iter(self._descriptions) - - def __len__(self): - return len(self._descriptions) + __len__, __iter__, __getitem__ = _make_sequence_methods("_descriptions") def __repr__(self): - return "".format(self._descriptions) + return "".format(self._descriptions) def __eq__(self, other): if not isinstance(other, AuthorityInformationAccess): @@ -290,9 +327,6 @@ class AuthorityInformationAccess(object): def __ne__(self, other): return not self == other - def __getitem__(self, idx): - return self._descriptions[idx] - def __hash__(self): return hash(tuple(self._descriptions)) @@ -419,14 +453,12 @@ class CRLDistributionPoints(object): self._distribution_points = distribution_points - def __iter__(self): - return iter(self._distribution_points) - - def __len__(self): - return len(self._distribution_points) + __len__, __iter__, __getitem__ = _make_sequence_methods( + "_distribution_points" + ) def __repr__(self): - return "".format(self._distribution_points) + return "".format(self._distribution_points) def __eq__(self, other): if not isinstance(other, CRLDistributionPoints): @@ -437,9 +469,6 @@ class CRLDistributionPoints(object): def __ne__(self, other): return not self == other - def __getitem__(self, idx): - return self._distribution_points[idx] - def __hash__(self): return hash(tuple(self._distribution_points)) @@ -460,14 +489,12 @@ class FreshestCRL(object): self._distribution_points = distribution_points - def __iter__(self): - return iter(self._distribution_points) - - def __len__(self): - return len(self._distribution_points) + __len__, __iter__, __getitem__ = _make_sequence_methods( + "_distribution_points" + ) def __repr__(self): - return "".format(self._distribution_points) + return "".format(self._distribution_points) def __eq__(self, other): if not isinstance(other, FreshestCRL): @@ -478,9 +505,6 @@ class FreshestCRL(object): def __ne__(self, other): return not self == other - def __getitem__(self, idx): - return self._distribution_points[idx] - def __hash__(self): return hash(tuple(self._distribution_points)) @@ -541,8 +565,8 @@ class DistributionPoint(object): def __repr__(self): return ( "".format(self) + "tive_name}, reasons={0.reasons}, crl_issuer={0.crl_issuer})>" + .format(self) ) def __eq__(self, other): @@ -666,14 +690,10 @@ class CertificatePolicies(object): self._policies = policies - def __iter__(self): - return iter(self._policies) - - def __len__(self): - return len(self._policies) + __len__, __iter__, __getitem__ = _make_sequence_methods("_policies") def __repr__(self): - return "".format(self._policies) + return "".format(self._policies) def __eq__(self, other): if not isinstance(other, CertificatePolicies): @@ -684,9 +704,6 @@ class CertificatePolicies(object): def __ne__(self, other): return not self == other - def __getitem__(self, idx): - return self._policies[idx] - def __hash__(self): return hash(tuple(self._policies)) @@ -827,14 +844,10 @@ class ExtendedKeyUsage(object): self._usages = usages - def __iter__(self): - return iter(self._usages) - - def __len__(self): - return len(self._usages) + __len__, __iter__, __getitem__ = _make_sequence_methods("_usages") def __repr__(self): - return "".format(self._usages) + return "".format(self._usages) def __eq__(self, other): if not isinstance(other, ExtendedKeyUsage): @@ -853,11 +866,41 @@ class ExtendedKeyUsage(object): class OCSPNoCheck(object): oid = ExtensionOID.OCSP_NO_CHECK + def __eq__(self, other): + if not isinstance(other, OCSPNoCheck): + return NotImplemented + + return True + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(OCSPNoCheck) + + def __repr__(self): + return "" + @utils.register_interface(ExtensionType) class PrecertPoison(object): oid = ExtensionOID.PRECERT_POISON + def __eq__(self, other): + if not isinstance(other, PrecertPoison): + return NotImplemented + + return True + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(PrecertPoison) + + def __repr__(self): + return "" + @utils.register_interface(ExtensionType) class TLSFeature(object): @@ -876,11 +919,7 @@ class TLSFeature(object): self._features = features - def __iter__(self): - return iter(self._features) - - def __len__(self): - return len(self._features) + __len__, __iter__, __getitem__ = _make_sequence_methods("_features") def __repr__(self): return "".format(self) @@ -891,9 +930,6 @@ class TLSFeature(object): return self._features == other._features - def __getitem__(self, idx): - return self._features[idx] - def __ne__(self, other): return not self == other @@ -1172,12 +1208,7 @@ class GeneralNames(object): ) self._general_names = general_names - - def __iter__(self): - return iter(self._general_names) - - def __len__(self): - return len(self._general_names) + __len__, __iter__, __getitem__ = _make_sequence_methods("_general_names") def get_values_for_type(self, type): # Return the value of each GeneralName, except for OtherName instances @@ -1189,7 +1220,7 @@ class GeneralNames(object): return list(objs) def __repr__(self): - return "".format(self._general_names) + return "".format(self._general_names) def __eq__(self, other): if not isinstance(other, GeneralNames): @@ -1200,9 +1231,6 @@ class GeneralNames(object): def __ne__(self, other): return not self == other - def __getitem__(self, idx): - return self._general_names[idx] - def __hash__(self): return hash(tuple(self._general_names)) @@ -1214,17 +1242,13 @@ class SubjectAlternativeName(object): def __init__(self, general_names): self._general_names = GeneralNames(general_names) - def __iter__(self): - return iter(self._general_names) - - def __len__(self): - return len(self._general_names) + __len__, __iter__, __getitem__ = _make_sequence_methods("_general_names") def get_values_for_type(self, type): return self._general_names.get_values_for_type(type) def __repr__(self): - return "".format(self._general_names) + return "".format(self._general_names) def __eq__(self, other): if not isinstance(other, SubjectAlternativeName): @@ -1232,9 +1256,6 @@ class SubjectAlternativeName(object): return self._general_names == other._general_names - def __getitem__(self, idx): - return self._general_names[idx] - def __ne__(self, other): return not self == other @@ -1249,17 +1270,13 @@ class IssuerAlternativeName(object): def __init__(self, general_names): self._general_names = GeneralNames(general_names) - def __iter__(self): - return iter(self._general_names) - - def __len__(self): - return len(self._general_names) + __len__, __iter__, __getitem__ = _make_sequence_methods("_general_names") def get_values_for_type(self, type): return self._general_names.get_values_for_type(type) def __repr__(self): - return "".format(self._general_names) + return "".format(self._general_names) def __eq__(self, other): if not isinstance(other, IssuerAlternativeName): @@ -1270,9 +1287,6 @@ class IssuerAlternativeName(object): def __ne__(self, other): return not self == other - def __getitem__(self, idx): - return self._general_names[idx] - def __hash__(self): return hash(self._general_names) @@ -1284,17 +1298,13 @@ class CertificateIssuer(object): def __init__(self, general_names): self._general_names = GeneralNames(general_names) - def __iter__(self): - return iter(self._general_names) - - def __len__(self): - return len(self._general_names) + __len__, __iter__, __getitem__ = _make_sequence_methods("_general_names") def get_values_for_type(self, type): return self._general_names.get_values_for_type(type) def __repr__(self): - return "".format(self._general_names) + return "".format(self._general_names) def __eq__(self, other): if not isinstance(other, CertificateIssuer): @@ -1305,9 +1315,6 @@ class CertificateIssuer(object): def __ne__(self, other): return not self == other - def __getitem__(self, idx): - return self._general_names[idx] - def __hash__(self): return hash(self._general_names) @@ -1323,7 +1330,7 @@ class CRLReason(object): self._reason = reason def __repr__(self): - return "".format(self._reason) + return "".format(self._reason) def __eq__(self, other): if not isinstance(other, CRLReason): @@ -1351,7 +1358,7 @@ class InvalidityDate(object): self._invalidity_date = invalidity_date def __repr__(self): - return "".format( + return "".format( self._invalidity_date ) @@ -1386,18 +1393,13 @@ class PrecertificateSignedCertificateTimestamps(object): ) self._signed_certificate_timestamps = signed_certificate_timestamps - def __iter__(self): - return iter(self._signed_certificate_timestamps) - - def __len__(self): - return len(self._signed_certificate_timestamps) - - def __getitem__(self, idx): - return self._signed_certificate_timestamps[idx] + __len__, __iter__, __getitem__ = _make_sequence_methods( + "_signed_certificate_timestamps" + ) def __repr__(self): return ( - "".format( + "".format( list(self) ) ) @@ -1446,6 +1448,136 @@ class OCSPNonce(object): nonce = utils.read_only_property("_nonce") +@utils.register_interface(ExtensionType) +class IssuingDistributionPoint(object): + oid = ExtensionOID.ISSUING_DISTRIBUTION_POINT + + def __init__(self, full_name, relative_name, only_contains_user_certs, + only_contains_ca_certs, only_some_reasons, indirect_crl, + only_contains_attribute_certs): + if ( + only_some_reasons and ( + not isinstance(only_some_reasons, frozenset) or not all( + isinstance(x, ReasonFlags) for x in only_some_reasons + ) + ) + ): + raise TypeError( + "only_some_reasons must be None or frozenset of ReasonFlags" + ) + + if only_some_reasons and ( + ReasonFlags.unspecified in only_some_reasons or + ReasonFlags.remove_from_crl in only_some_reasons + ): + raise ValueError( + "unspecified and remove_from_crl are not valid reasons in an " + "IssuingDistributionPoint" + ) + + if not ( + isinstance(only_contains_user_certs, bool) and + isinstance(only_contains_ca_certs, bool) and + isinstance(indirect_crl, bool) and + isinstance(only_contains_attribute_certs, bool) + ): + raise TypeError( + "only_contains_user_certs, only_contains_ca_certs, " + "indirect_crl and only_contains_attribute_certs " + "must all be boolean." + ) + + crl_constraints = [ + only_contains_user_certs, only_contains_ca_certs, + indirect_crl, only_contains_attribute_certs + ] + + if len([x for x in crl_constraints if x]) > 1: + raise ValueError( + "Only one of the following can be set to True: " + "only_contains_user_certs, only_contains_ca_certs, " + "indirect_crl, only_contains_attribute_certs" + ) + + if ( + not any([ + only_contains_user_certs, only_contains_ca_certs, + indirect_crl, only_contains_attribute_certs, full_name, + relative_name, only_some_reasons + ]) + ): + raise ValueError( + "Cannot create empty extension: " + "if only_contains_user_certs, only_contains_ca_certs, " + "indirect_crl, and only_contains_attribute_certs are all False" + ", then either full_name, relative_name, or only_some_reasons " + "must have a value." + ) + + self._only_contains_user_certs = only_contains_user_certs + self._only_contains_ca_certs = only_contains_ca_certs + self._indirect_crl = indirect_crl + self._only_contains_attribute_certs = only_contains_attribute_certs + self._only_some_reasons = only_some_reasons + self._full_name = full_name + self._relative_name = relative_name + + def __repr__(self): + return ( + "".format(self) + ) + + def __eq__(self, other): + if not isinstance(other, IssuingDistributionPoint): + return NotImplemented + + return ( + self.full_name == other.full_name and + self.relative_name == other.relative_name and + self.only_contains_user_certs == other.only_contains_user_certs and + self.only_contains_ca_certs == other.only_contains_ca_certs and + self.only_some_reasons == other.only_some_reasons and + self.indirect_crl == other.indirect_crl and + self.only_contains_attribute_certs == + other.only_contains_attribute_certs + ) + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(( + self.full_name, + self.relative_name, + self.only_contains_user_certs, + self.only_contains_ca_certs, + self.only_some_reasons, + self.indirect_crl, + self.only_contains_attribute_certs, + )) + + full_name = utils.read_only_property("_full_name") + relative_name = utils.read_only_property("_relative_name") + only_contains_user_certs = utils.read_only_property( + "_only_contains_user_certs" + ) + only_contains_ca_certs = utils.read_only_property( + "_only_contains_ca_certs" + ) + only_some_reasons = utils.read_only_property("_only_some_reasons") + indirect_crl = utils.read_only_property("_indirect_crl") + only_contains_attribute_certs = utils.read_only_property( + "_only_contains_attribute_certs" + ) + + @utils.register_interface(ExtensionType) class UnrecognizedExtension(object): def __init__(self, oid, value): diff --git a/server/www/packages/packages-linux/x64/cryptography/x509/general_name.py b/server/www/packages/packages-linux/x64/cryptography/x509/general_name.py index 26f389a..1233841 100644 --- a/server/www/packages/packages-linux/x64/cryptography/x509/general_name.py +++ b/server/www/packages/packages-linux/x64/cryptography/x509/general_name.py @@ -9,8 +9,6 @@ import ipaddress import warnings from email.utils import parseaddr -import idna - import six from six.moves import urllib_parse @@ -32,6 +30,20 @@ _GENERAL_NAMES = { } +def _lazy_import_idna(): + # Import idna lazily becase it allocates a decent amount of memory, and + # we're only using it in deprecated paths. + try: + import idna + return idna + except ImportError: + raise ImportError( + "idna is not installed, but a deprecated feature that requires it" + " was used. See: https://cryptography.io/en/latest/faq/#importe" + "rror-idna-is-not-installed" + ) + + class UnsupportedGeneralNameType(Exception): def __init__(self, msg, type): super(UnsupportedGeneralNameType, self).__init__(msg) @@ -60,7 +72,7 @@ class RFC822Name(object): "This means unicode characters should be encoded via " "idna. Support for passing unicode strings (aka U-label) " "will be removed in a future version.", - utils.DeprecatedIn21, + utils.PersistentlyDeprecated2017, stacklevel=2, ) else: @@ -83,6 +95,7 @@ class RFC822Name(object): return instance def _idna_encode(self, value): + idna = _lazy_import_idna() _, address = parseaddr(value) parts = address.split(u"@") return parts[0] + "@" + idna.encode(parts[1]).decode("ascii") @@ -104,6 +117,7 @@ class RFC822Name(object): def _idna_encode(value): + idna = _lazy_import_idna() # Retain prefixes '*.' for common/alt names and '.' for name constraints for prefix in ['*.', '.']: if value.startswith(prefix): @@ -125,7 +139,7 @@ class DNSName(object): "This means unicode characters should be encoded via " "idna. Support for passing unicode strings (aka U-label) " "will be removed in a future version.", - utils.DeprecatedIn21, + utils.PersistentlyDeprecated2017, stacklevel=2, ) else: @@ -170,7 +184,7 @@ class UniformResourceIdentifier(object): "This means unicode characters should be encoded via " "idna. Support for passing unicode strings (aka U-label) " " will be removed in a future version.", - utils.DeprecatedIn21, + utils.PersistentlyDeprecated2017, stacklevel=2, ) else: @@ -187,11 +201,12 @@ class UniformResourceIdentifier(object): return instance def _idna_encode(self, value): + idna = _lazy_import_idna() parsed = urllib_parse.urlparse(value) if parsed.port: netloc = ( idna.encode(parsed.hostname) + - ":{0}".format(parsed.port).encode("ascii") + ":{}".format(parsed.port).encode("ascii") ).decode("ascii") else: netloc = idna.encode(parsed.hostname).decode("ascii") @@ -235,7 +250,7 @@ class DirectoryName(object): value = utils.read_only_property("_value") def __repr__(self): - return "".format(self.value) + return "".format(self.value) def __eq__(self, other): if not isinstance(other, DirectoryName): @@ -261,7 +276,7 @@ class RegisteredID(object): value = utils.read_only_property("_value") def __repr__(self): - return "".format(self.value) + return "".format(self.value) def __eq__(self, other): if not isinstance(other, RegisteredID): @@ -299,7 +314,7 @@ class IPAddress(object): value = utils.read_only_property("_value") def __repr__(self): - return "".format(self.value) + return "".format(self.value) def __eq__(self, other): if not isinstance(other, IPAddress): @@ -329,7 +344,7 @@ class OtherName(object): value = utils.read_only_property("_value") def __repr__(self): - return "".format( + return "".format( self.type_id, self.value) def __eq__(self, other): diff --git a/server/www/packages/packages-linux/x64/cryptography/x509/name.py b/server/www/packages/packages-linux/x64/cryptography/x509/name.py index 5548eda..ca2a175 100644 --- a/server/www/packages/packages-linux/x64/cryptography/x509/name.py +++ b/server/www/packages/packages-linux/x64/cryptography/x509/name.py @@ -36,6 +36,41 @@ _NAMEOID_DEFAULT_TYPE = { NameOID.DOMAIN_COMPONENT: _ASN1Type.IA5String, } +#: Short attribute names from RFC 4514: +#: https://tools.ietf.org/html/rfc4514#page-7 +_NAMEOID_TO_NAME = { + NameOID.COMMON_NAME: 'CN', + NameOID.LOCALITY_NAME: 'L', + NameOID.STATE_OR_PROVINCE_NAME: 'ST', + NameOID.ORGANIZATION_NAME: 'O', + NameOID.ORGANIZATIONAL_UNIT_NAME: 'OU', + NameOID.COUNTRY_NAME: 'C', + NameOID.STREET_ADDRESS: 'STREET', + NameOID.DOMAIN_COMPONENT: 'DC', + NameOID.USER_ID: 'UID', +} + + +def _escape_dn_value(val): + """Escape special characters in RFC4514 Distinguished Name value.""" + + # See https://tools.ietf.org/html/rfc4514#section-2.4 + val = val.replace('\\', '\\\\') + val = val.replace('"', '\\"') + val = val.replace('+', '\\+') + val = val.replace(',', '\\,') + val = val.replace(';', '\\;') + val = val.replace('<', '\\<') + val = val.replace('>', '\\>') + val = val.replace('\0', '\\00') + + if val[0] in ('#', ' '): + val = '\\' + val + if val[-1] == ' ': + val = val[:-1] + '\\ ' + + return val + class NameAttribute(object): def __init__(self, oid, value, _type=_SENTINEL): @@ -80,6 +115,16 @@ class NameAttribute(object): oid = utils.read_only_property("_oid") value = utils.read_only_property("_value") + def rfc4514_string(self): + """ + Format as RFC4514 Distinguished Name string. + + Use short attribute name if available, otherwise fall back to OID + dotted string. + """ + key = _NAMEOID_TO_NAME.get(self.oid, self.oid.dotted_string) + return '%s=%s' % (key, _escape_dn_value(self.value)) + def __eq__(self, other): if not isinstance(other, NameAttribute): return NotImplemented @@ -117,6 +162,15 @@ class RelativeDistinguishedName(object): def get_attributes_for_oid(self, oid): return [i for i in self if i.oid == oid] + def rfc4514_string(self): + """ + Format as RFC4514 Distinguished Name string. + + Within each RDN, attributes are joined by '+', although that is rarely + used in certificates. + """ + return '+'.join(attr.rfc4514_string() for attr in self._attributes) + def __eq__(self, other): if not isinstance(other, RelativeDistinguishedName): return NotImplemented @@ -136,7 +190,7 @@ class RelativeDistinguishedName(object): return len(self._attributes) def __repr__(self): - return "".format(list(self)) + return "".format(self.rfc4514_string()) class Name(object): @@ -154,6 +208,18 @@ class Name(object): " or a list RelativeDistinguishedName" ) + def rfc4514_string(self): + """ + Format as RFC4514 Distinguished Name string. + For example 'CN=foobar.com,O=Foo Corp,C=US' + + An X.509 name is a two-level structure: a list of sets of attributes. + Each list element is separated by ',' and within each list element, set + elements are separated by '+'. The latter is almost never used in + real world certificates. + """ + return ','.join(attr.rfc4514_string() for attr in self._attributes) + def get_attributes_for_oid(self, oid): return [i for i in self if i.oid == oid] @@ -187,4 +253,7 @@ class Name(object): return sum(len(rdn) for rdn in self._attributes) def __repr__(self): - return "".format(list(self)) + if six.PY2: + return "".format(self.rfc4514_string().encode('utf8')) + else: + return "".format(self.rfc4514_string()) diff --git a/server/www/packages/packages-linux/x64/cryptography/x509/ocsp.py b/server/www/packages/packages-linux/x64/cryptography/x509/ocsp.py index 2b0b1dc..b15063d 100644 --- a/server/www/packages/packages-linux/x64/cryptography/x509/ocsp.py +++ b/server/www/packages/packages-linux/x64/cryptography/x509/ocsp.py @@ -12,8 +12,9 @@ import six from cryptography import x509 from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import ed25519, ed448 from cryptography.x509.base import ( - _UNIX_EPOCH, _convert_to_naive_utc_time, _reject_duplicate_extension + _EARLIEST_UTC_TIME, _convert_to_naive_utc_time, _reject_duplicate_extension ) @@ -154,9 +155,9 @@ class _SingleResponse(object): raise TypeError("revocation_time must be a datetime object") revocation_time = _convert_to_naive_utc_time(revocation_time) - if revocation_time <= _UNIX_EPOCH: - raise ValueError('The revocation_time must be after the unix' - ' epoch (1970 January 1).') + if revocation_time < _EARLIEST_UTC_TIME: + raise ValueError('The revocation_time must be on or after' + ' 1950 January 1.') if ( revocation_reason is not None and @@ -241,7 +242,13 @@ class OCSPResponseBuilder(object): if self._responder_id is None: raise ValueError("You must add a responder_id before signing") - if not isinstance(algorithm, hashes.HashAlgorithm): + if isinstance(private_key, + (ed25519.Ed25519PrivateKey, ed448.Ed448PrivateKey)): + if algorithm is not None: + raise ValueError( + "algorithm must be None when signing via ed25519 or ed448" + ) + elif not isinstance(algorithm, hashes.HashAlgorithm): raise TypeError("Algorithm must be a registered hash algorithm.") return backend.create_ocsp_response( @@ -314,6 +321,12 @@ class OCSPResponse(object): The ObjectIdentifier of the signature algorithm """ + @abc.abstractproperty + def signature_hash_algorithm(self): + """ + Returns a HashAlgorithm corresponding to the type of the digest signed + """ + @abc.abstractproperty def signature(self): """ diff --git a/server/www/packages/packages-linux/x64/cryptography/x509/oid.py b/server/www/packages/packages-linux/x64/cryptography/x509/oid.py index ec19007..c1e5dc5 100644 --- a/server/www/packages/packages-linux/x64/cryptography/x509/oid.py +++ b/server/www/packages/packages-linux/x64/cryptography/x509/oid.py @@ -96,6 +96,8 @@ class SignatureAlgorithmOID(object): DSA_WITH_SHA1 = ObjectIdentifier("1.2.840.10040.4.3") DSA_WITH_SHA224 = ObjectIdentifier("2.16.840.1.101.3.4.3.1") DSA_WITH_SHA256 = ObjectIdentifier("2.16.840.1.101.3.4.3.2") + ED25519 = ObjectIdentifier("1.3.101.112") + ED448 = ObjectIdentifier("1.3.101.113") _SIG_OIDS_TO_HASH = { @@ -113,7 +115,9 @@ _SIG_OIDS_TO_HASH = { SignatureAlgorithmOID.ECDSA_WITH_SHA512: hashes.SHA512(), SignatureAlgorithmOID.DSA_WITH_SHA1: hashes.SHA1(), SignatureAlgorithmOID.DSA_WITH_SHA224: hashes.SHA224(), - SignatureAlgorithmOID.DSA_WITH_SHA256: hashes.SHA256() + SignatureAlgorithmOID.DSA_WITH_SHA256: hashes.SHA256(), + SignatureAlgorithmOID.ED25519: None, + SignatureAlgorithmOID.ED448: None, } @@ -181,6 +185,8 @@ _OID_NAMES = { SignatureAlgorithmOID.DSA_WITH_SHA1: "dsa-with-sha1", SignatureAlgorithmOID.DSA_WITH_SHA224: "dsa-with-sha224", SignatureAlgorithmOID.DSA_WITH_SHA256: "dsa-with-sha256", + SignatureAlgorithmOID.ED25519: "ed25519", + SignatureAlgorithmOID.ED448: "ed448", ExtendedKeyUsageOID.SERVER_AUTH: "serverAuth", ExtendedKeyUsageOID.CLIENT_AUTH: "clientAuth", ExtendedKeyUsageOID.CODE_SIGNING: "codeSigning", @@ -196,6 +202,7 @@ _OID_NAMES = { ExtensionOID.PRECERT_SIGNED_CERTIFICATE_TIMESTAMPS: ( "signedCertificateTimestampList" ), + ExtensionOID.PRECERT_POISON: "ctPoison", CRLEntryExtensionOID.CRL_REASON: "cRLReason", CRLEntryExtensionOID.INVALIDITY_DATE: "invalidityDate", CRLEntryExtensionOID.CERTIFICATE_ISSUER: "certificateIssuer", diff --git a/server/www/packages/packages-linux/x64/ldap3/__init__.py b/server/www/packages/packages-linux/x64/ldap3/__init__.py index f6f1666..c352994 100644 --- a/server/www/packages/packages-linux/x64/ldap3/__init__.py +++ b/server/www/packages/packages-linux/x64/ldap3/__init__.py @@ -89,6 +89,7 @@ SCHEMA = 'SCHEMA' ALL = 'ALL' OFFLINE_EDIR_8_8_8 = 'EDIR_8_8_8' +OFFLINE_EDIR_9_1_4 = 'EDIR_9_1_4' OFFLINE_AD_2012_R2 = 'AD_2012_R2' OFFLINE_SLAPD_2_4 = 'SLAPD_2_4' OFFLINE_DS389_1_3_3 = 'DS389_1_3_3' diff --git a/server/www/packages/packages-linux/x64/ldap3/abstract/__init__.py b/server/www/packages/packages-linux/x64/ldap3/abstract/__init__.py index c40f838..22ea8c9 100644 --- a/server/www/packages/packages-linux/x64/ldap3/abstract/__init__.py +++ b/server/www/packages/packages-linux/x64/ldap3/abstract/__init__.py @@ -5,7 +5,7 @@ # # Author: Giovanni Cannata # -# Copyright 2014 - 2018 Giovanni Cannata +# Copyright 2014 - 2019 Giovanni Cannata # # This file is part of ldap3. # diff --git a/server/www/packages/packages-linux/x64/ldap3/abstract/attrDef.py b/server/www/packages/packages-linux/x64/ldap3/abstract/attrDef.py index d954e25..21d8f3e 100644 --- a/server/www/packages/packages-linux/x64/ldap3/abstract/attrDef.py +++ b/server/www/packages/packages-linux/x64/ldap3/abstract/attrDef.py @@ -5,7 +5,7 @@ # # Author: Giovanni Cannata # -# Copyright 2014 - 2018 Giovanni Cannata +# Copyright 2014 - 2019 Giovanni Cannata # # This file is part of ldap3. diff --git a/server/www/packages/packages-linux/x64/ldap3/abstract/attribute.py b/server/www/packages/packages-linux/x64/ldap3/abstract/attribute.py index 24f682c..434ee12 100644 --- a/server/www/packages/packages-linux/x64/ldap3/abstract/attribute.py +++ b/server/www/packages/packages-linux/x64/ldap3/abstract/attribute.py @@ -5,7 +5,7 @@ # # Author: Giovanni Cannata # -# Copyright 2014 - 2018 Giovanni Cannata +# Copyright 2014 - 2019 Giovanni Cannata # # This file is part of ldap3. # @@ -174,7 +174,7 @@ class WritableAttribute(Attribute): log(PROTOCOL, 'adding %r to <%s> attribute in <%s> entry', values, self.key, self.entry.entry_dn) # new value for attribute to commit with a MODIFY_ADD if self.entry._state._initial_status == STATUS_VIRTUAL: - error_message = 'cannot add an attribute value in a new entry' + error_message = 'cannot perform a modify operation in a new entry' if log_enabled(ERROR): log(ERROR, '%s for <%s>', error_message, self) raise LDAPCursorError(error_message) diff --git a/server/www/packages/packages-linux/x64/ldap3/abstract/cursor.py b/server/www/packages/packages-linux/x64/ldap3/abstract/cursor.py index 275a384..ad774a1 100644 --- a/server/www/packages/packages-linux/x64/ldap3/abstract/cursor.py +++ b/server/www/packages/packages-linux/x64/ldap3/abstract/cursor.py @@ -1,904 +1,906 @@ -""" -""" - -# Created on 2014.01.06 -# -# Author: Giovanni Cannata -# -# Copyright 2014 - 2018 Giovanni Cannata -# -# This file is part of ldap3. -# -# ldap3 is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# ldap3 is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with ldap3 in the COPYING and COPYING.LESSER files. -# If not, see . -from collections import namedtuple -from copy import deepcopy -from datetime import datetime -from os import linesep -from time import sleep - -from . import STATUS_VIRTUAL, STATUS_READ, STATUS_WRITABLE -from .. import SUBTREE, LEVEL, DEREF_ALWAYS, DEREF_NEVER, BASE, SEQUENCE_TYPES, STRING_TYPES, get_config_parameter -from ..abstract import STATUS_PENDING_CHANGES -from .attribute import Attribute, OperationalAttribute, WritableAttribute -from .attrDef import AttrDef -from .objectDef import ObjectDef -from .entry import Entry, WritableEntry -from ..core.exceptions import LDAPCursorError, LDAPObjectDereferenceError -from ..core.results import RESULT_SUCCESS -from ..utils.ciDict import CaseInsensitiveWithAliasDict -from ..utils.dn import safe_dn, safe_rdn -from ..utils.conv import to_raw -from ..utils.config import get_config_parameter -from ..utils.log import log, log_enabled, ERROR, BASIC, PROTOCOL, EXTENDED -from ..protocol.oid import ATTRIBUTE_DIRECTORY_OPERATION, ATTRIBUTE_DISTRIBUTED_OPERATION, ATTRIBUTE_DSA_OPERATION, CLASS_AUXILIARY - -Operation = namedtuple('Operation', ('request', 'result', 'response')) - - -def _ret_search_value(value): - return value[0] + '=' + value[1:] if value[0] in '<>~' and value[1] != '=' else value - - -def _create_query_dict(query_text): - """ - Create a dictionary with query key:value definitions - query_text is a comma delimited key:value sequence - """ - query_dict = dict() - if query_text: - for arg_value_str in query_text.split(','): - if ':' in arg_value_str: - arg_value_list = arg_value_str.split(':') - query_dict[arg_value_list[0].strip()] = arg_value_list[1].strip() - - return query_dict - - -class Cursor(object): - # entry_class and attribute_class define the type of entry and attribute used by the cursor - # entry_initial_status defines the initial status of a entry - # entry_class = Entry, must be defined in subclasses - # attribute_class = Attribute, must be defined in subclasses - # entry_initial_status = STATUS, must be defined in subclasses - - def __init__(self, connection, object_def, get_operational_attributes=False, attributes=None, controls=None, auxiliary_class=None): - conf_attributes_excluded_from_object_def = [v.lower() for v in get_config_parameter('ATTRIBUTES_EXCLUDED_FROM_OBJECT_DEF')] - self.connection = connection - self.get_operational_attributes = get_operational_attributes - if connection._deferred_bind or connection._deferred_open: # probably a lazy connection, tries to bind - connection._fire_deferred() - - if isinstance(object_def, (STRING_TYPES, SEQUENCE_TYPES)): - object_def = ObjectDef(object_def, connection.server.schema, auxiliary_class=auxiliary_class) - self.definition = object_def - if attributes: # checks if requested attributes are defined in ObjectDef - not_defined_attributes = [] - if isinstance(attributes, STRING_TYPES): - attributes = [attributes] - - for attribute in attributes: - if attribute not in self.definition._attributes and attribute.lower() not in conf_attributes_excluded_from_object_def: - not_defined_attributes.append(attribute) - - if not_defined_attributes: - error_message = 'Attributes \'%s\' non in definition' % ', '.join(not_defined_attributes) - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise LDAPCursorError(error_message) - - self.attributes = set(attributes) if attributes else set([attr.name for attr in self.definition]) - self.controls = controls - self.execution_time = None - self.entries = [] - self.schema = self.connection.server.schema - self._do_not_reset = False # used for refreshing entry in entry_refresh() without removing all entries from the Cursor - self._operation_history = list() # a list storing all the requests, results and responses for the last cursor operation - - def __repr__(self): - r = 'CURSOR : ' + self.__class__.__name__ + linesep - r += 'CONN : ' + str(self.connection) + linesep - r += 'DEFS : ' + ', '.join(self.definition._object_class) - if self.definition._auxiliary_class: - r += ' [AUX: ' + ', '.join(self.definition._auxiliary_class) + ']' - r += linesep - # for attr_def in sorted(self.definition): - # r += (attr_def.key if attr_def.key == attr_def.name else (attr_def.key + ' <' + attr_def.name + '>')) + ', ' - # if r[-2] == ',': - # r = r[:-2] - # r += ']' + linesep - if hasattr(self, 'attributes'): - r += 'ATTRS : ' + repr(sorted(self.attributes)) + (' [OPERATIONAL]' if self.get_operational_attributes else '') + linesep - if isinstance(self, Reader): - if hasattr(self, 'base'): - r += 'BASE : ' + repr(self.base) + (' [SUB]' if self.sub_tree else ' [LEVEL]') + linesep - if hasattr(self, '_query') and self._query: - r += 'QUERY : ' + repr(self._query) + ('' if '(' in self._query else (' [AND]' if self.components_in_and else ' [OR]')) + linesep - if hasattr(self, 'validated_query') and self.validated_query: - r += 'PARSED : ' + repr(self.validated_query) + ('' if '(' in self._query else (' [AND]' if self.components_in_and else ' [OR]')) + linesep - if hasattr(self, 'query_filter') and self.query_filter: - r += 'FILTER : ' + repr(self.query_filter) + linesep - - if hasattr(self, 'execution_time') and self.execution_time: - r += 'ENTRIES: ' + str(len(self.entries)) - r += ' [executed at: ' + str(self.execution_time.isoformat()) + ']' + linesep - - if self.failed: - r += 'LAST OPERATION FAILED [' + str(len(self.errors)) + ' failure' + ('s' if len(self.errors) > 1 else '') + ' at operation' + ('s ' if len(self.errors) > 1 else ' ') + ', '.join([str(i) for i, error in enumerate(self.operations) if error.result['result'] != RESULT_SUCCESS]) + ']' - - return r - - def __str__(self): - return self.__repr__() - - def __iter__(self): - return self.entries.__iter__() - - def __getitem__(self, item): - """Return indexed item, if index is not found then try to sequentially search in DN of entries. - If only one entry is found return it else raise a KeyError exception. The exception message - includes the number of entries that matches, if less than 10 entries match then show the DNs - in the exception message. - """ - try: - return self.entries[item] - except TypeError: - pass - - if isinstance(item, STRING_TYPES): - found = self.match_dn(item) - - if len(found) == 1: - return found[0] - elif len(found) > 1: - error_message = 'Multiple entries found: %d entries match the text in dn' % len(found) + ('' if len(found) > 10 else (' [' + '; '.join([e.entry_dn for e in found]) + ']')) - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise KeyError(error_message) - - error_message = 'no entry found' - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise KeyError(error_message) - - def __len__(self): - return len(self.entries) - - if str is not bytes: # Python 3 - def __bool__(self): # needed to make the cursor appears as existing in "if cursor:" even if there are no entries - return True - else: # Python 2 - def __nonzero__(self): - return True - - def _get_attributes(self, response, attr_defs, entry): - """Assign the result of the LDAP query to the Entry object dictionary. - - If the optional 'post_query' callable is present in the AttrDef it is called with each value of the attribute and the callable result is stored in the attribute. - - Returns the default value for missing attributes. - If the 'dereference_dn' in AttrDef is a ObjectDef then the attribute values are treated as distinguished name and the relevant entry is retrieved and stored in the attribute value. - - """ - conf_operational_attribute_prefix = get_config_parameter('ABSTRACTION_OPERATIONAL_ATTRIBUTE_PREFIX') - conf_attributes_excluded_from_object_def = [v.lower() for v in get_config_parameter('ATTRIBUTES_EXCLUDED_FROM_OBJECT_DEF')] - attributes = CaseInsensitiveWithAliasDict() - used_attribute_names = set() - for attr in attr_defs: - attr_def = attr_defs[attr] - attribute_name = None - for attr_name in response['attributes']: - if attr_def.name.lower() == attr_name.lower(): - attribute_name = attr_name - break - - if attribute_name or attr_def.default is not NotImplemented: # attribute value found in result or default value present - NotImplemented allows use of None as default - attribute = self.attribute_class(attr_def, entry, self) - attribute.response = response - attribute.raw_values = response['raw_attributes'][attribute_name] if attribute_name else None - if attr_def.post_query and attr_def.name in response['attributes'] and response['raw_attributes'] != list(): - attribute.values = attr_def.post_query(attr_def.key, response['attributes'][attribute_name]) - else: - if attr_def.default is NotImplemented or (attribute_name and response['raw_attributes'][attribute_name] != list()): - attribute.values = response['attributes'][attribute_name] - else: - attribute.values = attr_def.default if isinstance(attr_def.default, SEQUENCE_TYPES) else [attr_def.default] - if not isinstance(attribute.values, list): # force attribute values to list (if attribute is single-valued) - attribute.values = [attribute.values] - if attr_def.dereference_dn: # try to get object referenced in value - if attribute.values: - temp_reader = Reader(self.connection, attr_def.dereference_dn, base='', get_operational_attributes=self.get_operational_attributes, controls=self.controls) - temp_values = [] - for element in attribute.values: - if entry.entry_dn != element: - temp_values.append(temp_reader.search_object(element)) - else: - error_message = 'object %s is referencing itself in the \'%s\' attribute' % (entry.entry_dn, attribute.definition.name) - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise LDAPObjectDereferenceError(error_message) - del temp_reader # remove the temporary Reader - attribute.values = temp_values - attributes[attribute.key] = attribute - if attribute.other_names: - attributes.set_alias(attribute.key, attribute.other_names) - if attr_def.other_names: - attributes.set_alias(attribute.key, attr_def.other_names) - used_attribute_names.add(attribute_name) - - if self.attributes: - used_attribute_names.update(self.attributes) - - for attribute_name in response['attributes']: - if attribute_name not in used_attribute_names: - operational_attribute = False - # check if the type is an operational attribute - if attribute_name in self.schema.attribute_types: - if self.schema.attribute_types[attribute_name].no_user_modification or self.schema.attribute_types[attribute_name].usage in [ATTRIBUTE_DIRECTORY_OPERATION, ATTRIBUTE_DISTRIBUTED_OPERATION, ATTRIBUTE_DSA_OPERATION]: - operational_attribute = True - else: - operational_attribute = True - if not operational_attribute and attribute_name not in attr_defs and attribute_name.lower() not in conf_attributes_excluded_from_object_def: - error_message = 'attribute \'%s\' not in object class \'%s\' for entry %s' % (attribute_name, ', '.join(entry.entry_definition._object_class), entry.entry_dn) - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise LDAPCursorError(error_message) - attribute = OperationalAttribute(AttrDef(conf_operational_attribute_prefix + attribute_name), entry, self) - attribute.raw_values = response['raw_attributes'][attribute_name] - attribute.values = response['attributes'][attribute_name] if isinstance(response['attributes'][attribute_name], SEQUENCE_TYPES) else [response['attributes'][attribute_name]] - if (conf_operational_attribute_prefix + attribute_name) not in attributes: - attributes[conf_operational_attribute_prefix + attribute_name] = attribute - - return attributes - - def match_dn(self, dn): - """Return entries with text in DN""" - matched = [] - for entry in self.entries: - if dn.lower() in entry.entry_dn.lower(): - matched.append(entry) - return matched - - def match(self, attributes, value): - """Return entries with text in one of the specified attributes""" - matched = [] - if not isinstance(attributes, SEQUENCE_TYPES): - attributes = [attributes] - - for entry in self.entries: - found = False - for attribute in attributes: - if attribute in entry: - for attr_value in entry[attribute].values: - if hasattr(attr_value, 'lower') and hasattr(value, 'lower') and value.lower() in attr_value.lower(): - found = True - elif value == attr_value: - found = True - if found: - matched.append(entry) - break - if found: - break - # checks raw values, tries to convert value to byte - raw_value = to_raw(value) - if isinstance(raw_value, (bytes, bytearray)): - for attr_value in entry[attribute].raw_values: - if hasattr(attr_value, 'lower') and hasattr(raw_value, 'lower') and raw_value.lower() in attr_value.lower(): - found = True - elif raw_value == attr_value: - found = True - if found: - matched.append(entry) - break - if found: - break - return matched - - def _create_entry(self, response): - if not response['type'] == 'searchResEntry': - return None - - entry = self.entry_class(response['dn'], self) # define an Entry (writable or readonly), as specified in the cursor definition - entry._state.attributes = self._get_attributes(response, self.definition._attributes, entry) - entry._state.entry_raw_attributes = deepcopy(response['raw_attributes']) - - entry._state.response = response - entry._state.read_time = datetime.now() - entry._state.set_status(self.entry_initial_status) - for attr in entry: # returns the whole attribute object - entry.__dict__[attr.key] = attr - - return entry - - def _execute_query(self, query_scope, attributes): - if not self.connection: - error_message = 'no connection established' - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise LDAPCursorError(error_message) - old_query_filter = None - if query_scope == BASE: # requesting a single object so an always-valid filter is set - if hasattr(self, 'query_filter'): # only Reader has a query filter - old_query_filter = self.query_filter - self.query_filter = '(objectclass=*)' - else: - self._create_query_filter() - if log_enabled(PROTOCOL): - log(PROTOCOL, 'executing query - base: %s - filter: %s - scope: %s for <%s>', self.base, self.query_filter, query_scope, self) - with self.connection: - result = self.connection.search(search_base=self.base, - search_filter=self.query_filter, - search_scope=query_scope, - dereference_aliases=self.dereference_aliases, - attributes=attributes if attributes else list(self.attributes), - get_operational_attributes=self.get_operational_attributes, - controls=self.controls) - if not self.connection.strategy.sync: - response, result, request = self.connection.get_response(result, get_request=True) - else: - response = self.connection.response - result = self.connection.result - request = self.connection.request - - self._store_operation_in_history(request, result, response) - - if self._do_not_reset: # trick to not remove entries when using _refresh() - return self._create_entry(response[0]) - - self.entries = [] - for r in response: - entry = self._create_entry(r) - if entry is not None: - self.entries.append(entry) - if 'objectClass' in entry: - for object_class in entry.objectClass: - if self.schema.object_classes[object_class].kind == CLASS_AUXILIARY and object_class not in self.definition._auxiliary_class: - # add auxiliary class to object definition - self.definition._auxiliary_class.append(object_class) - self.definition._populate_attr_defs(object_class) - self.execution_time = datetime.now() - - if old_query_filter: # requesting a single object so an always-valid filter is set - self.query_filter = old_query_filter - - def remove(self, entry): - if log_enabled(PROTOCOL): - log(PROTOCOL, 'removing entry <%s> in <%s>', entry, self) - self.entries.remove(entry) - - def _reset_history(self): - self._operation_history = list() - - def _store_operation_in_history(self, request, result, response): - self._operation_history.append(Operation(request, result, response)) - - @property - def operations(self): - return self._operation_history - - @property - def errors(self): - return [error for error in self._operation_history if error.result['result'] != RESULT_SUCCESS] - - @property - def failed(self): - if hasattr(self, '_operation_history'): - return any([error.result['result'] != RESULT_SUCCESS for error in self._operation_history]) - - -class Reader(Cursor): - """Reader object to perform searches: - - :param connection: the LDAP connection object to use - :type connection: LDAPConnection - :param object_def: the ObjectDef of the LDAP object returned - :type object_def: ObjectDef - :param query: the simplified query (will be transformed in an LDAP filter) - :type query: str - :param base: starting base of the search - :type base: str - :param components_in_and: specify if assertions in the query must all be satisfied or not (AND/OR) - :type components_in_and: bool - :param sub_tree: specify if the search must be performed ad Single Level (False) or Whole SubTree (True) - :type sub_tree: bool - :param get_operational_attributes: specify if operational attributes are returned or not - :type get_operational_attributes: bool - :param controls: controls to be used in search - :type controls: tuple - - """ - entry_class = Entry # entries are read_only - attribute_class = Attribute # attributes are read_only - entry_initial_status = STATUS_READ - - def __init__(self, connection, object_def, base, query='', components_in_and=True, sub_tree=True, get_operational_attributes=False, attributes=None, controls=None, auxiliary_class=None): - Cursor.__init__(self, connection, object_def, get_operational_attributes, attributes, controls, auxiliary_class) - self._components_in_and = components_in_and - self.sub_tree = sub_tree - self._query = query - self.base = base - self.dereference_aliases = DEREF_ALWAYS - self.validated_query = None - self._query_dict = dict() - self._validated_query_dict = dict() - self.query_filter = None - self.reset() - - if log_enabled(BASIC): - log(BASIC, 'instantiated Reader Cursor: <%r>', self) - - @property - def query(self): - return self._query - - @query.setter - def query(self, value): - self._query = value - self.reset() - - @property - def components_in_and(self): - return self._components_in_and - - @components_in_and.setter - def components_in_and(self, value): - self._components_in_and = value - self.reset() - - def clear(self): - """Clear the Reader search parameters - - """ - self.dereference_aliases = DEREF_ALWAYS - self._reset_history() - - def reset(self): - """Clear all the Reader parameters - - """ - self.clear() - self.validated_query = None - self._query_dict = dict() - self._validated_query_dict = dict() - self.execution_time = None - self.query_filter = None - self.entries = [] - self._create_query_filter() - - def _validate_query(self): - """Processes the text query and verifies that the requested friendly names are in the Reader dictionary - If the AttrDef has a 'validate' property the callable is executed and if it returns False an Exception is raised - - """ - if not self._query_dict: - self._query_dict = _create_query_dict(self._query) - - query = '' - for d in sorted(self._query_dict): - attr = d[1:] if d[0] in '&|' else d - for attr_def in self.definition: - if ''.join(attr.split()).lower() == attr_def.key.lower(): - attr = attr_def.key - break - if attr in self.definition: - vals = sorted(self._query_dict[d].split(';')) - - query += (d[0] + attr if d[0] in '&|' else attr) + ': ' - for val in vals: - val = val.strip() - val_not = True if val[0] == '!' else False - val_search_operator = '=' # default - if val_not: - if val[1:].lstrip()[0] not in '=<>~': - value = val[1:].lstrip() - else: - val_search_operator = val[1:].lstrip()[0] - value = val[1:].lstrip()[1:] - else: - if val[0] not in '=<>~': - value = val.lstrip() - else: - val_search_operator = val[0] - value = val[1:].lstrip() - - if self.definition[attr].validate: - validated = self.definition[attr].validate(value) # returns True, False or a value to substitute to the actual values - if validated is False: - error_message = 'validation failed for attribute %s and value %s' % (d, val) - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise LDAPCursorError(error_message) - elif validated is not True: # a valid LDAP value equivalent to the actual values - value = validated - if val_not: - query += '!' + val_search_operator + str(value) - else: - query += val_search_operator + str(value) - - query += ';' - query = query[:-1] + ', ' - else: - error_message = 'attribute \'%s\' not in definition' % attr - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise LDAPCursorError(error_message) - self.validated_query = query[:-2] - self._validated_query_dict = _create_query_dict(self.validated_query) - - def _create_query_filter(self): - """Converts the query dictionary to the filter text""" - self.query_filter = '' - - if self.definition._object_class: - self.query_filter += '(&' - if isinstance(self.definition._object_class, SEQUENCE_TYPES) and len(self.definition._object_class) == 1: - self.query_filter += '(objectClass=' + self.definition._object_class[0] + ')' - elif isinstance(self.definition._object_class, SEQUENCE_TYPES): - self.query_filter += '(&' - for object_class in self.definition._object_class: - self.query_filter += '(objectClass=' + object_class + ')' - self.query_filter += ')' - else: - error_message = 'object class must be a string or a list' - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise LDAPCursorError(error_message) - - if self._query and self._query.startswith('(') and self._query.endswith(')'): # query is already an LDAP filter - if 'objectclass' not in self._query.lower(): - self.query_filter += self._query + ')' # if objectclass not in filter adds from definition - else: - self.query_filter = self._query - return - elif self._query: # if a simplified filter is present - if not self.components_in_and: - self.query_filter += '(|' - elif not self.definition._object_class: - self.query_filter += '(&' - - self._validate_query() - - attr_counter = 0 - for attr in sorted(self._validated_query_dict): - attr_counter += 1 - multi = True if ';' in self._validated_query_dict[attr] else False - vals = sorted(self._validated_query_dict[attr].split(';')) - attr_def = self.definition[attr[1:]] if attr[0] in '&|' else self.definition[attr] - if attr_def.pre_query: - modvals = [] - for val in vals: - modvals.append(val[0] + attr_def.pre_query(attr_def.key, val[1:])) - vals = modvals - if multi: - if attr[0] in '&|': - self.query_filter += '(' + attr[0] - else: - self.query_filter += '(|' - - for val in vals: - if val[0] == '!': - self.query_filter += '(!(' + attr_def.name + _ret_search_value(val[1:]) + '))' - else: - self.query_filter += '(' + attr_def.name + _ret_search_value(val) + ')' - if multi: - self.query_filter += ')' - - if not self.components_in_and: - self.query_filter += '))' - else: - self.query_filter += ')' - - if not self.definition._object_class and attr_counter == 1: # removes unneeded starting filter - self.query_filter = self.query_filter[2: -1] - - if self.query_filter == '(|)' or self.query_filter == '(&)': # removes empty filter - self.query_filter = '' - else: # no query, remove unneeded leading (& - self.query_filter = self.query_filter[2:] - - def search(self, attributes=None): - """Perform the LDAP search - - :return: Entries found in search - - """ - self.clear() - query_scope = SUBTREE if self.sub_tree else LEVEL - if log_enabled(PROTOCOL): - log(PROTOCOL, 'performing search in <%s>', self) - self._execute_query(query_scope, attributes) - - return self.entries - - def search_object(self, entry_dn=None, attributes=None): # base must be a single dn - """Perform the LDAP search operation SINGLE_OBJECT scope - - :return: Entry found in search - - """ - if log_enabled(PROTOCOL): - log(PROTOCOL, 'performing object search in <%s>', self) - self.clear() - if entry_dn: - old_base = self.base - self.base = entry_dn - self._execute_query(BASE, attributes) - self.base = old_base - else: - self._execute_query(BASE, attributes) - - return self.entries[0] if len(self.entries) > 0 else None - - def search_level(self, attributes=None): - """Perform the LDAP search operation with SINGLE_LEVEL scope - - :return: Entries found in search - - """ - if log_enabled(PROTOCOL): - log(PROTOCOL, 'performing single level search in <%s>', self) - self.clear() - self._execute_query(LEVEL, attributes) - - return self.entries - - def search_subtree(self, attributes=None): - """Perform the LDAP search operation WHOLE_SUBTREE scope - - :return: Entries found in search - - """ - if log_enabled(PROTOCOL): - log(PROTOCOL, 'performing whole subtree search in <%s>', self) - self.clear() - self._execute_query(SUBTREE, attributes) - - return self.entries - - def _entries_generator(self, responses): - for response in responses: - yield self._create_entry(response) - - def search_paged(self, paged_size, paged_criticality=True, generator=True, attributes=None): - """Perform a paged search, can be called as an Iterator - - :param attributes: optional attributes to search - :param paged_size: number of entries returned in each search - :type paged_size: int - :param paged_criticality: specify if server must not execute the search if it is not capable of paging searches - :type paged_criticality: bool - :param generator: if True the paged searches are executed while generating the entries, - if False all the paged searches are execute before returning the generator - :type generator: bool - :return: Entries found in search - - """ - if log_enabled(PROTOCOL): - log(PROTOCOL, 'performing paged search in <%s> with paged size %s', self, str(paged_size)) - if not self.connection: - error_message = 'no connection established' - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise LDAPCursorError(error_message) - - self.clear() - self._create_query_filter() - self.entries = [] - self.execution_time = datetime.now() - response = self.connection.extend.standard.paged_search(search_base=self.base, - search_filter=self.query_filter, - search_scope=SUBTREE if self.sub_tree else LEVEL, - dereference_aliases=self.dereference_aliases, - attributes=attributes if attributes else self.attributes, - get_operational_attributes=self.get_operational_attributes, - controls=self.controls, - paged_size=paged_size, - paged_criticality=paged_criticality, - generator=generator) - if generator: - return self._entries_generator(response) - else: - return list(self._entries_generator(response)) - - -class Writer(Cursor): - entry_class = WritableEntry - attribute_class = WritableAttribute - entry_initial_status = STATUS_WRITABLE - - @staticmethod - def from_cursor(cursor, connection=None, object_def=None, custom_validator=None): - if connection is None: - connection = cursor.connection - if object_def is None: - object_def = cursor.definition - writer = Writer(connection, object_def, attributes=cursor.attributes) - for entry in cursor.entries: - if isinstance(cursor, Reader): - entry.entry_writable(object_def, writer, custom_validator=custom_validator) - elif isinstance(cursor, Writer): - pass - else: - error_message = 'unknown cursor type %s' % str(type(cursor)) - if log_enabled(ERROR): - log(ERROR, '%s', error_message) - raise LDAPCursorError(error_message) - writer.execution_time = cursor.execution_time - if log_enabled(BASIC): - log(BASIC, 'instantiated Writer Cursor <%r> from cursor <%r>', writer, cursor) - return writer - - @staticmethod - def from_response(connection, object_def, response=None): - if response is None: - if not connection.strategy.sync: - error_message = 'with asynchronous strategies response must be specified' - if log_enabled(ERROR): - log(ERROR, '%s', error_message) - raise LDAPCursorError(error_message) - elif connection.response: - response = connection.response - else: - error_message = 'response not present' - if log_enabled(ERROR): - log(ERROR, '%s', error_message) - raise LDAPCursorError(error_message) - writer = Writer(connection, object_def) - - for resp in response: - if resp['type'] == 'searchResEntry': - entry = writer._create_entry(resp) - writer.entries.append(entry) - if log_enabled(BASIC): - log(BASIC, 'instantiated Writer Cursor <%r> from response', writer) - return writer - - def __init__(self, connection, object_def, get_operational_attributes=False, attributes=None, controls=None, auxiliary_class=None): - Cursor.__init__(self, connection, object_def, get_operational_attributes, attributes, controls, auxiliary_class) - self.dereference_aliases = DEREF_NEVER - - if log_enabled(BASIC): - log(BASIC, 'instantiated Writer Cursor: <%r>', self) - - def commit(self, refresh=True): - if log_enabled(PROTOCOL): - log(PROTOCOL, 'committed changes for <%s>', self) - self._reset_history() - successful = True - for entry in self.entries: - if not entry.entry_commit_changes(refresh=refresh, controls=self.controls, clear_history=False): - successful = False - - self.execution_time = datetime.now() - - return successful - - def discard(self): - if log_enabled(PROTOCOL): - log(PROTOCOL, 'discarded changes for <%s>', self) - for entry in self.entries: - entry.entry_discard_changes() - - def _refresh_object(self, entry_dn, attributes=None, tries=4, seconds=2, controls=None): # base must be a single dn - """Performs the LDAP search operation SINGLE_OBJECT scope - - :return: Entry found in search - - """ - if log_enabled(PROTOCOL): - log(PROTOCOL, 'refreshing object <%s> for <%s>', entry_dn, self) - if not self.connection: - error_message = 'no connection established' - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise LDAPCursorError(error_message) - - response = [] - with self.connection: - counter = 0 - while counter < tries: - result = self.connection.search(search_base=entry_dn, - search_filter='(objectclass=*)', - search_scope=BASE, - dereference_aliases=DEREF_NEVER, - attributes=attributes if attributes else self.attributes, - get_operational_attributes=self.get_operational_attributes, - controls=controls) - if not self.connection.strategy.sync: - response, result, request = self.connection.get_response(result, get_request=True) - else: - response = self.connection.response - result = self.connection.result - request = self.connection.request - - if result['result'] in [RESULT_SUCCESS]: - break - sleep(seconds) - counter += 1 - self._store_operation_in_history(request, result, response) - - if len(response) == 1: - return self._create_entry(response[0]) - elif len(response) == 0: - return None - - error_message = 'more than 1 entry returned for a single object search' - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise LDAPCursorError(error_message) - - def new(self, dn): - if log_enabled(BASIC): - log(BASIC, 'creating new entry <%s> for <%s>', dn, self) - dn = safe_dn(dn) - for entry in self.entries: # checks if dn is already used in an cursor entry - if entry.entry_dn == dn: - error_message = 'dn already present in cursor' - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise LDAPCursorError(error_message) - rdns = safe_rdn(dn, decompose=True) - entry = self.entry_class(dn, self) # defines a new empty Entry - for attr in entry.entry_mandatory_attributes: # defines all mandatory attributes as virtual - entry._state.attributes[attr] = self.attribute_class(entry._state.definition[attr], entry, self) - entry.__dict__[attr] = entry._state.attributes[attr] - entry.objectclass.set(self.definition._object_class) - for rdn in rdns: # adds virtual attributes from rdns in entry name (should be more than one with + syntax) - if rdn[0] in entry._state.definition._attributes: - rdn_name = entry._state.definition._attributes[rdn[0]].name # normalize case folding - if rdn_name not in entry._state.attributes: - entry._state.attributes[rdn_name] = self.attribute_class(entry._state.definition[rdn_name], entry, self) - entry.__dict__[rdn_name] = entry._state.attributes[rdn_name] - entry.__dict__[rdn_name].set(rdn[1]) - else: - error_message = 'rdn type \'%s\' not in object class definition' % rdn[0] - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise LDAPCursorError(error_message) - entry._state.set_status(STATUS_VIRTUAL) # set intial status - entry._state.set_status(STATUS_PENDING_CHANGES) # tries to change status to PENDING_CHANGES. If mandatory attributes are missing status is reverted to MANDATORY_MISSING - self.entries.append(entry) - return entry - - def refresh_entry(self, entry, tries=4, seconds=2): - conf_operational_attribute_prefix = get_config_parameter('ABSTRACTION_OPERATIONAL_ATTRIBUTE_PREFIX') - - self._do_not_reset = True - attr_list = [] - if log_enabled(PROTOCOL): - log(PROTOCOL, 'refreshing entry <%s> for <%s>', entry, self) - for attr in entry._state.attributes: # check friendly attribute name in AttrDef, do not check operational attributes - if attr.lower().startswith(conf_operational_attribute_prefix.lower()): - continue - if entry._state.definition[attr].name: - attr_list.append(entry._state.definition[attr].name) - else: - attr_list.append(entry._state.definition[attr].key) - - temp_entry = self._refresh_object(entry.entry_dn, attr_list, tries, seconds=seconds) # if any attributes is added adds only to the entry not to the definition - self._do_not_reset = False - if temp_entry: - temp_entry._state.origin = entry._state.origin - entry.__dict__.clear() - entry.__dict__['_state'] = temp_entry._state - for attr in entry._state.attributes: # returns the attribute key - entry.__dict__[attr] = entry._state.attributes[attr] - - for attr in entry.entry_attributes: # if any attribute of the class was deleted makes it virtual - if attr not in entry._state.attributes and attr in entry.entry_definition._attributes: - entry._state.attributes[attr] = WritableAttribute(entry.entry_definition[attr], entry, self) - entry.__dict__[attr] = entry._state.attributes[attr] - entry._state.set_status(entry._state._initial_status) - return True - return False +""" +""" + +# Created on 2014.01.06 +# +# Author: Giovanni Cannata +# +# Copyright 2014 - 2019 Giovanni Cannata +# +# This file is part of ldap3. +# +# ldap3 is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ldap3 is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with ldap3 in the COPYING and COPYING.LESSER files. +# If not, see . +from collections import namedtuple +from copy import deepcopy +from datetime import datetime +from os import linesep +from time import sleep + +from . import STATUS_VIRTUAL, STATUS_READ, STATUS_WRITABLE +from .. import SUBTREE, LEVEL, DEREF_ALWAYS, DEREF_NEVER, BASE, SEQUENCE_TYPES, STRING_TYPES, get_config_parameter +from ..abstract import STATUS_PENDING_CHANGES +from .attribute import Attribute, OperationalAttribute, WritableAttribute +from .attrDef import AttrDef +from .objectDef import ObjectDef +from .entry import Entry, WritableEntry +from ..core.exceptions import LDAPCursorError, LDAPObjectDereferenceError +from ..core.results import RESULT_SUCCESS +from ..utils.ciDict import CaseInsensitiveWithAliasDict +from ..utils.dn import safe_dn, safe_rdn +from ..utils.conv import to_raw +from ..utils.config import get_config_parameter +from ..utils.log import log, log_enabled, ERROR, BASIC, PROTOCOL, EXTENDED +from ..protocol.oid import ATTRIBUTE_DIRECTORY_OPERATION, ATTRIBUTE_DISTRIBUTED_OPERATION, ATTRIBUTE_DSA_OPERATION, CLASS_AUXILIARY + +Operation = namedtuple('Operation', ('request', 'result', 'response')) + + +def _ret_search_value(value): + return value[0] + '=' + value[1:] if value[0] in '<>~' and value[1] != '=' else value + + +def _create_query_dict(query_text): + """ + Create a dictionary with query key:value definitions + query_text is a comma delimited key:value sequence + """ + query_dict = dict() + if query_text: + for arg_value_str in query_text.split(','): + if ':' in arg_value_str: + arg_value_list = arg_value_str.split(':') + query_dict[arg_value_list[0].strip()] = arg_value_list[1].strip() + + return query_dict + + +class Cursor(object): + # entry_class and attribute_class define the type of entry and attribute used by the cursor + # entry_initial_status defines the initial status of a entry + # entry_class = Entry, must be defined in subclasses + # attribute_class = Attribute, must be defined in subclasses + # entry_initial_status = STATUS, must be defined in subclasses + + def __init__(self, connection, object_def, get_operational_attributes=False, attributes=None, controls=None, auxiliary_class=None): + conf_attributes_excluded_from_object_def = [v.lower() for v in get_config_parameter('ATTRIBUTES_EXCLUDED_FROM_OBJECT_DEF')] + self.connection = connection + self.get_operational_attributes = get_operational_attributes + if connection._deferred_bind or connection._deferred_open: # probably a lazy connection, tries to bind + connection._fire_deferred() + + if isinstance(object_def, (STRING_TYPES, SEQUENCE_TYPES)): + if connection.closed: # try to open connection if closed to read schema + connection.bind() + object_def = ObjectDef(object_def, connection.server.schema, auxiliary_class=auxiliary_class) + self.definition = object_def + if attributes: # checks if requested attributes are defined in ObjectDef + not_defined_attributes = [] + if isinstance(attributes, STRING_TYPES): + attributes = [attributes] + + for attribute in attributes: + if attribute not in self.definition._attributes and attribute.lower() not in conf_attributes_excluded_from_object_def: + not_defined_attributes.append(attribute) + + if not_defined_attributes: + error_message = 'Attributes \'%s\' non in definition' % ', '.join(not_defined_attributes) + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise LDAPCursorError(error_message) + + self.attributes = set(attributes) if attributes else set([attr.name for attr in self.definition]) + self.controls = controls + self.execution_time = None + self.entries = [] + self.schema = self.connection.server.schema + self._do_not_reset = False # used for refreshing entry in entry_refresh() without removing all entries from the Cursor + self._operation_history = list() # a list storing all the requests, results and responses for the last cursor operation + + def __repr__(self): + r = 'CURSOR : ' + self.__class__.__name__ + linesep + r += 'CONN : ' + str(self.connection) + linesep + r += 'DEFS : ' + ', '.join(self.definition._object_class) + if self.definition._auxiliary_class: + r += ' [AUX: ' + ', '.join(self.definition._auxiliary_class) + ']' + r += linesep + # for attr_def in sorted(self.definition): + # r += (attr_def.key if attr_def.key == attr_def.name else (attr_def.key + ' <' + attr_def.name + '>')) + ', ' + # if r[-2] == ',': + # r = r[:-2] + # r += ']' + linesep + if hasattr(self, 'attributes'): + r += 'ATTRS : ' + repr(sorted(self.attributes)) + (' [OPERATIONAL]' if self.get_operational_attributes else '') + linesep + if isinstance(self, Reader): + if hasattr(self, 'base'): + r += 'BASE : ' + repr(self.base) + (' [SUB]' if self.sub_tree else ' [LEVEL]') + linesep + if hasattr(self, '_query') and self._query: + r += 'QUERY : ' + repr(self._query) + ('' if '(' in self._query else (' [AND]' if self.components_in_and else ' [OR]')) + linesep + if hasattr(self, 'validated_query') and self.validated_query: + r += 'PARSED : ' + repr(self.validated_query) + ('' if '(' in self._query else (' [AND]' if self.components_in_and else ' [OR]')) + linesep + if hasattr(self, 'query_filter') and self.query_filter: + r += 'FILTER : ' + repr(self.query_filter) + linesep + + if hasattr(self, 'execution_time') and self.execution_time: + r += 'ENTRIES: ' + str(len(self.entries)) + r += ' [executed at: ' + str(self.execution_time.isoformat()) + ']' + linesep + + if self.failed: + r += 'LAST OPERATION FAILED [' + str(len(self.errors)) + ' failure' + ('s' if len(self.errors) > 1 else '') + ' at operation' + ('s ' if len(self.errors) > 1 else ' ') + ', '.join([str(i) for i, error in enumerate(self.operations) if error.result['result'] != RESULT_SUCCESS]) + ']' + + return r + + def __str__(self): + return self.__repr__() + + def __iter__(self): + return self.entries.__iter__() + + def __getitem__(self, item): + """Return indexed item, if index is not found then try to sequentially search in DN of entries. + If only one entry is found return it else raise a KeyError exception. The exception message + includes the number of entries that matches, if less than 10 entries match then show the DNs + in the exception message. + """ + try: + return self.entries[item] + except TypeError: + pass + + if isinstance(item, STRING_TYPES): + found = self.match_dn(item) + + if len(found) == 1: + return found[0] + elif len(found) > 1: + error_message = 'Multiple entries found: %d entries match the text in dn' % len(found) + ('' if len(found) > 10 else (' [' + '; '.join([e.entry_dn for e in found]) + ']')) + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise KeyError(error_message) + + error_message = 'no entry found' + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise KeyError(error_message) + + def __len__(self): + return len(self.entries) + + if str is not bytes: # Python 3 + def __bool__(self): # needed to make the cursor appears as existing in "if cursor:" even if there are no entries + return True + else: # Python 2 + def __nonzero__(self): + return True + + def _get_attributes(self, response, attr_defs, entry): + """Assign the result of the LDAP query to the Entry object dictionary. + + If the optional 'post_query' callable is present in the AttrDef it is called with each value of the attribute and the callable result is stored in the attribute. + + Returns the default value for missing attributes. + If the 'dereference_dn' in AttrDef is a ObjectDef then the attribute values are treated as distinguished name and the relevant entry is retrieved and stored in the attribute value. + + """ + conf_operational_attribute_prefix = get_config_parameter('ABSTRACTION_OPERATIONAL_ATTRIBUTE_PREFIX') + conf_attributes_excluded_from_object_def = [v.lower() for v in get_config_parameter('ATTRIBUTES_EXCLUDED_FROM_OBJECT_DEF')] + attributes = CaseInsensitiveWithAliasDict() + used_attribute_names = set() + for attr in attr_defs: + attr_def = attr_defs[attr] + attribute_name = None + for attr_name in response['attributes']: + if attr_def.name.lower() == attr_name.lower(): + attribute_name = attr_name + break + + if attribute_name or attr_def.default is not NotImplemented: # attribute value found in result or default value present - NotImplemented allows use of None as default + attribute = self.attribute_class(attr_def, entry, self) + attribute.response = response + attribute.raw_values = response['raw_attributes'][attribute_name] if attribute_name else None + if attr_def.post_query and attr_def.name in response['attributes'] and response['raw_attributes'] != list(): + attribute.values = attr_def.post_query(attr_def.key, response['attributes'][attribute_name]) + else: + if attr_def.default is NotImplemented or (attribute_name and response['raw_attributes'][attribute_name] != list()): + attribute.values = response['attributes'][attribute_name] + else: + attribute.values = attr_def.default if isinstance(attr_def.default, SEQUENCE_TYPES) else [attr_def.default] + if not isinstance(attribute.values, list): # force attribute values to list (if attribute is single-valued) + attribute.values = [attribute.values] + if attr_def.dereference_dn: # try to get object referenced in value + if attribute.values: + temp_reader = Reader(self.connection, attr_def.dereference_dn, base='', get_operational_attributes=self.get_operational_attributes, controls=self.controls) + temp_values = [] + for element in attribute.values: + if entry.entry_dn != element: + temp_values.append(temp_reader.search_object(element)) + else: + error_message = 'object %s is referencing itself in the \'%s\' attribute' % (entry.entry_dn, attribute.definition.name) + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise LDAPObjectDereferenceError(error_message) + del temp_reader # remove the temporary Reader + attribute.values = temp_values + attributes[attribute.key] = attribute + if attribute.other_names: + attributes.set_alias(attribute.key, attribute.other_names) + if attr_def.other_names: + attributes.set_alias(attribute.key, attr_def.other_names) + used_attribute_names.add(attribute_name) + + if self.attributes: + used_attribute_names.update(self.attributes) + + for attribute_name in response['attributes']: + if attribute_name not in used_attribute_names: + operational_attribute = False + # check if the type is an operational attribute + if attribute_name in self.schema.attribute_types: + if self.schema.attribute_types[attribute_name].no_user_modification or self.schema.attribute_types[attribute_name].usage in [ATTRIBUTE_DIRECTORY_OPERATION, ATTRIBUTE_DISTRIBUTED_OPERATION, ATTRIBUTE_DSA_OPERATION]: + operational_attribute = True + else: + operational_attribute = True + if not operational_attribute and attribute_name not in attr_defs and attribute_name.lower() not in conf_attributes_excluded_from_object_def: + error_message = 'attribute \'%s\' not in object class \'%s\' for entry %s' % (attribute_name, ', '.join(entry.entry_definition._object_class), entry.entry_dn) + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise LDAPCursorError(error_message) + attribute = OperationalAttribute(AttrDef(conf_operational_attribute_prefix + attribute_name), entry, self) + attribute.raw_values = response['raw_attributes'][attribute_name] + attribute.values = response['attributes'][attribute_name] if isinstance(response['attributes'][attribute_name], SEQUENCE_TYPES) else [response['attributes'][attribute_name]] + if (conf_operational_attribute_prefix + attribute_name) not in attributes: + attributes[conf_operational_attribute_prefix + attribute_name] = attribute + + return attributes + + def match_dn(self, dn): + """Return entries with text in DN""" + matched = [] + for entry in self.entries: + if dn.lower() in entry.entry_dn.lower(): + matched.append(entry) + return matched + + def match(self, attributes, value): + """Return entries with text in one of the specified attributes""" + matched = [] + if not isinstance(attributes, SEQUENCE_TYPES): + attributes = [attributes] + + for entry in self.entries: + found = False + for attribute in attributes: + if attribute in entry: + for attr_value in entry[attribute].values: + if hasattr(attr_value, 'lower') and hasattr(value, 'lower') and value.lower() in attr_value.lower(): + found = True + elif value == attr_value: + found = True + if found: + matched.append(entry) + break + if found: + break + # checks raw values, tries to convert value to byte + raw_value = to_raw(value) + if isinstance(raw_value, (bytes, bytearray)): + for attr_value in entry[attribute].raw_values: + if hasattr(attr_value, 'lower') and hasattr(raw_value, 'lower') and raw_value.lower() in attr_value.lower(): + found = True + elif raw_value == attr_value: + found = True + if found: + matched.append(entry) + break + if found: + break + return matched + + def _create_entry(self, response): + if not response['type'] == 'searchResEntry': + return None + + entry = self.entry_class(response['dn'], self) # define an Entry (writable or readonly), as specified in the cursor definition + entry._state.attributes = self._get_attributes(response, self.definition._attributes, entry) + entry._state.raw_attributes = deepcopy(response['raw_attributes']) + + entry._state.response = response + entry._state.read_time = datetime.now() + entry._state.set_status(self.entry_initial_status) + for attr in entry: # returns the whole attribute object + entry.__dict__[attr.key] = attr + + return entry + + def _execute_query(self, query_scope, attributes): + if not self.connection: + error_message = 'no connection established' + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise LDAPCursorError(error_message) + old_query_filter = None + if query_scope == BASE: # requesting a single object so an always-valid filter is set + if hasattr(self, 'query_filter'): # only Reader has a query filter + old_query_filter = self.query_filter + self.query_filter = '(objectclass=*)' + else: + self._create_query_filter() + if log_enabled(PROTOCOL): + log(PROTOCOL, 'executing query - base: %s - filter: %s - scope: %s for <%s>', self.base, self.query_filter, query_scope, self) + with self.connection: + result = self.connection.search(search_base=self.base, + search_filter=self.query_filter, + search_scope=query_scope, + dereference_aliases=self.dereference_aliases, + attributes=attributes if attributes else list(self.attributes), + get_operational_attributes=self.get_operational_attributes, + controls=self.controls) + if not self.connection.strategy.sync: + response, result, request = self.connection.get_response(result, get_request=True) + else: + response = self.connection.response + result = self.connection.result + request = self.connection.request + + self._store_operation_in_history(request, result, response) + + if self._do_not_reset: # trick to not remove entries when using _refresh() + return self._create_entry(response[0]) + + self.entries = [] + for r in response: + entry = self._create_entry(r) + if entry is not None: + self.entries.append(entry) + if 'objectClass' in entry: + for object_class in entry.objectClass: + if self.schema and self.schema.object_classes[object_class].kind == CLASS_AUXILIARY and object_class not in self.definition._auxiliary_class: + # add auxiliary class to object definition + self.definition._auxiliary_class.append(object_class) + self.definition._populate_attr_defs(object_class) + self.execution_time = datetime.now() + + if old_query_filter: # requesting a single object so an always-valid filter is set + self.query_filter = old_query_filter + + def remove(self, entry): + if log_enabled(PROTOCOL): + log(PROTOCOL, 'removing entry <%s> in <%s>', entry, self) + self.entries.remove(entry) + + def _reset_history(self): + self._operation_history = list() + + def _store_operation_in_history(self, request, result, response): + self._operation_history.append(Operation(request, result, response)) + + @property + def operations(self): + return self._operation_history + + @property + def errors(self): + return [error for error in self._operation_history if error.result['result'] != RESULT_SUCCESS] + + @property + def failed(self): + if hasattr(self, '_operation_history'): + return any([error.result['result'] != RESULT_SUCCESS for error in self._operation_history]) + + +class Reader(Cursor): + """Reader object to perform searches: + + :param connection: the LDAP connection object to use + :type connection: LDAPConnection + :param object_def: the ObjectDef of the LDAP object returned + :type object_def: ObjectDef + :param query: the simplified query (will be transformed in an LDAP filter) + :type query: str + :param base: starting base of the search + :type base: str + :param components_in_and: specify if assertions in the query must all be satisfied or not (AND/OR) + :type components_in_and: bool + :param sub_tree: specify if the search must be performed ad Single Level (False) or Whole SubTree (True) + :type sub_tree: bool + :param get_operational_attributes: specify if operational attributes are returned or not + :type get_operational_attributes: bool + :param controls: controls to be used in search + :type controls: tuple + + """ + entry_class = Entry # entries are read_only + attribute_class = Attribute # attributes are read_only + entry_initial_status = STATUS_READ + + def __init__(self, connection, object_def, base, query='', components_in_and=True, sub_tree=True, get_operational_attributes=False, attributes=None, controls=None, auxiliary_class=None): + Cursor.__init__(self, connection, object_def, get_operational_attributes, attributes, controls, auxiliary_class) + self._components_in_and = components_in_and + self.sub_tree = sub_tree + self._query = query + self.base = base + self.dereference_aliases = DEREF_ALWAYS + self.validated_query = None + self._query_dict = dict() + self._validated_query_dict = dict() + self.query_filter = None + self.reset() + + if log_enabled(BASIC): + log(BASIC, 'instantiated Reader Cursor: <%r>', self) + + @property + def query(self): + return self._query + + @query.setter + def query(self, value): + self._query = value + self.reset() + + @property + def components_in_and(self): + return self._components_in_and + + @components_in_and.setter + def components_in_and(self, value): + self._components_in_and = value + self.reset() + + def clear(self): + """Clear the Reader search parameters + + """ + self.dereference_aliases = DEREF_ALWAYS + self._reset_history() + + def reset(self): + """Clear all the Reader parameters + + """ + self.clear() + self.validated_query = None + self._query_dict = dict() + self._validated_query_dict = dict() + self.execution_time = None + self.query_filter = None + self.entries = [] + self._create_query_filter() + + def _validate_query(self): + """Processes the text query and verifies that the requested friendly names are in the Reader dictionary + If the AttrDef has a 'validate' property the callable is executed and if it returns False an Exception is raised + + """ + if not self._query_dict: + self._query_dict = _create_query_dict(self._query) + + query = '' + for d in sorted(self._query_dict): + attr = d[1:] if d[0] in '&|' else d + for attr_def in self.definition: + if ''.join(attr.split()).lower() == attr_def.key.lower(): + attr = attr_def.key + break + if attr in self.definition: + vals = sorted(self._query_dict[d].split(';')) + + query += (d[0] + attr if d[0] in '&|' else attr) + ': ' + for val in vals: + val = val.strip() + val_not = True if val[0] == '!' else False + val_search_operator = '=' # default + if val_not: + if val[1:].lstrip()[0] not in '=<>~': + value = val[1:].lstrip() + else: + val_search_operator = val[1:].lstrip()[0] + value = val[1:].lstrip()[1:] + else: + if val[0] not in '=<>~': + value = val.lstrip() + else: + val_search_operator = val[0] + value = val[1:].lstrip() + + if self.definition[attr].validate: + validated = self.definition[attr].validate(value) # returns True, False or a value to substitute to the actual values + if validated is False: + error_message = 'validation failed for attribute %s and value %s' % (d, val) + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise LDAPCursorError(error_message) + elif validated is not True: # a valid LDAP value equivalent to the actual values + value = validated + if val_not: + query += '!' + val_search_operator + str(value) + else: + query += val_search_operator + str(value) + + query += ';' + query = query[:-1] + ', ' + else: + error_message = 'attribute \'%s\' not in definition' % attr + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise LDAPCursorError(error_message) + self.validated_query = query[:-2] + self._validated_query_dict = _create_query_dict(self.validated_query) + + def _create_query_filter(self): + """Converts the query dictionary to the filter text""" + self.query_filter = '' + + if self.definition._object_class: + self.query_filter += '(&' + if isinstance(self.definition._object_class, SEQUENCE_TYPES) and len(self.definition._object_class) == 1: + self.query_filter += '(objectClass=' + self.definition._object_class[0] + ')' + elif isinstance(self.definition._object_class, SEQUENCE_TYPES): + self.query_filter += '(&' + for object_class in self.definition._object_class: + self.query_filter += '(objectClass=' + object_class + ')' + self.query_filter += ')' + else: + error_message = 'object class must be a string or a list' + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise LDAPCursorError(error_message) + + if self._query and self._query.startswith('(') and self._query.endswith(')'): # query is already an LDAP filter + if 'objectclass' not in self._query.lower(): + self.query_filter += self._query + ')' # if objectclass not in filter adds from definition + else: + self.query_filter = self._query + return + elif self._query: # if a simplified filter is present + if not self.components_in_and: + self.query_filter += '(|' + elif not self.definition._object_class: + self.query_filter += '(&' + + self._validate_query() + + attr_counter = 0 + for attr in sorted(self._validated_query_dict): + attr_counter += 1 + multi = True if ';' in self._validated_query_dict[attr] else False + vals = sorted(self._validated_query_dict[attr].split(';')) + attr_def = self.definition[attr[1:]] if attr[0] in '&|' else self.definition[attr] + if attr_def.pre_query: + modvals = [] + for val in vals: + modvals.append(val[0] + attr_def.pre_query(attr_def.key, val[1:])) + vals = modvals + if multi: + if attr[0] in '&|': + self.query_filter += '(' + attr[0] + else: + self.query_filter += '(|' + + for val in vals: + if val[0] == '!': + self.query_filter += '(!(' + attr_def.name + _ret_search_value(val[1:]) + '))' + else: + self.query_filter += '(' + attr_def.name + _ret_search_value(val) + ')' + if multi: + self.query_filter += ')' + + if not self.components_in_and: + self.query_filter += '))' + else: + self.query_filter += ')' + + if not self.definition._object_class and attr_counter == 1: # removes unneeded starting filter + self.query_filter = self.query_filter[2: -1] + + if self.query_filter == '(|)' or self.query_filter == '(&)': # removes empty filter + self.query_filter = '' + else: # no query, remove unneeded leading (& + self.query_filter = self.query_filter[2:] + + def search(self, attributes=None): + """Perform the LDAP search + + :return: Entries found in search + + """ + self.clear() + query_scope = SUBTREE if self.sub_tree else LEVEL + if log_enabled(PROTOCOL): + log(PROTOCOL, 'performing search in <%s>', self) + self._execute_query(query_scope, attributes) + + return self.entries + + def search_object(self, entry_dn=None, attributes=None): # base must be a single dn + """Perform the LDAP search operation SINGLE_OBJECT scope + + :return: Entry found in search + + """ + if log_enabled(PROTOCOL): + log(PROTOCOL, 'performing object search in <%s>', self) + self.clear() + if entry_dn: + old_base = self.base + self.base = entry_dn + self._execute_query(BASE, attributes) + self.base = old_base + else: + self._execute_query(BASE, attributes) + + return self.entries[0] if len(self.entries) > 0 else None + + def search_level(self, attributes=None): + """Perform the LDAP search operation with SINGLE_LEVEL scope + + :return: Entries found in search + + """ + if log_enabled(PROTOCOL): + log(PROTOCOL, 'performing single level search in <%s>', self) + self.clear() + self._execute_query(LEVEL, attributes) + + return self.entries + + def search_subtree(self, attributes=None): + """Perform the LDAP search operation WHOLE_SUBTREE scope + + :return: Entries found in search + + """ + if log_enabled(PROTOCOL): + log(PROTOCOL, 'performing whole subtree search in <%s>', self) + self.clear() + self._execute_query(SUBTREE, attributes) + + return self.entries + + def _entries_generator(self, responses): + for response in responses: + yield self._create_entry(response) + + def search_paged(self, paged_size, paged_criticality=True, generator=True, attributes=None): + """Perform a paged search, can be called as an Iterator + + :param attributes: optional attributes to search + :param paged_size: number of entries returned in each search + :type paged_size: int + :param paged_criticality: specify if server must not execute the search if it is not capable of paging searches + :type paged_criticality: bool + :param generator: if True the paged searches are executed while generating the entries, + if False all the paged searches are execute before returning the generator + :type generator: bool + :return: Entries found in search + + """ + if log_enabled(PROTOCOL): + log(PROTOCOL, 'performing paged search in <%s> with paged size %s', self, str(paged_size)) + if not self.connection: + error_message = 'no connection established' + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise LDAPCursorError(error_message) + + self.clear() + self._create_query_filter() + self.entries = [] + self.execution_time = datetime.now() + response = self.connection.extend.standard.paged_search(search_base=self.base, + search_filter=self.query_filter, + search_scope=SUBTREE if self.sub_tree else LEVEL, + dereference_aliases=self.dereference_aliases, + attributes=attributes if attributes else self.attributes, + get_operational_attributes=self.get_operational_attributes, + controls=self.controls, + paged_size=paged_size, + paged_criticality=paged_criticality, + generator=generator) + if generator: + return self._entries_generator(response) + else: + return list(self._entries_generator(response)) + + +class Writer(Cursor): + entry_class = WritableEntry + attribute_class = WritableAttribute + entry_initial_status = STATUS_WRITABLE + + @staticmethod + def from_cursor(cursor, connection=None, object_def=None, custom_validator=None): + if connection is None: + connection = cursor.connection + if object_def is None: + object_def = cursor.definition + writer = Writer(connection, object_def, attributes=cursor.attributes) + for entry in cursor.entries: + if isinstance(cursor, Reader): + entry.entry_writable(object_def, writer, custom_validator=custom_validator) + elif isinstance(cursor, Writer): + pass + else: + error_message = 'unknown cursor type %s' % str(type(cursor)) + if log_enabled(ERROR): + log(ERROR, '%s', error_message) + raise LDAPCursorError(error_message) + writer.execution_time = cursor.execution_time + if log_enabled(BASIC): + log(BASIC, 'instantiated Writer Cursor <%r> from cursor <%r>', writer, cursor) + return writer + + @staticmethod + def from_response(connection, object_def, response=None): + if response is None: + if not connection.strategy.sync: + error_message = 'with asynchronous strategies response must be specified' + if log_enabled(ERROR): + log(ERROR, '%s', error_message) + raise LDAPCursorError(error_message) + elif connection.response: + response = connection.response + else: + error_message = 'response not present' + if log_enabled(ERROR): + log(ERROR, '%s', error_message) + raise LDAPCursorError(error_message) + writer = Writer(connection, object_def) + + for resp in response: + if resp['type'] == 'searchResEntry': + entry = writer._create_entry(resp) + writer.entries.append(entry) + if log_enabled(BASIC): + log(BASIC, 'instantiated Writer Cursor <%r> from response', writer) + return writer + + def __init__(self, connection, object_def, get_operational_attributes=False, attributes=None, controls=None, auxiliary_class=None): + Cursor.__init__(self, connection, object_def, get_operational_attributes, attributes, controls, auxiliary_class) + self.dereference_aliases = DEREF_NEVER + + if log_enabled(BASIC): + log(BASIC, 'instantiated Writer Cursor: <%r>', self) + + def commit(self, refresh=True): + if log_enabled(PROTOCOL): + log(PROTOCOL, 'committed changes for <%s>', self) + self._reset_history() + successful = True + for entry in self.entries: + if not entry.entry_commit_changes(refresh=refresh, controls=self.controls, clear_history=False): + successful = False + + self.execution_time = datetime.now() + + return successful + + def discard(self): + if log_enabled(PROTOCOL): + log(PROTOCOL, 'discarded changes for <%s>', self) + for entry in self.entries: + entry.entry_discard_changes() + + def _refresh_object(self, entry_dn, attributes=None, tries=4, seconds=2, controls=None): # base must be a single dn + """Performs the LDAP search operation SINGLE_OBJECT scope + + :return: Entry found in search + + """ + if log_enabled(PROTOCOL): + log(PROTOCOL, 'refreshing object <%s> for <%s>', entry_dn, self) + if not self.connection: + error_message = 'no connection established' + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise LDAPCursorError(error_message) + + response = [] + with self.connection: + counter = 0 + while counter < tries: + result = self.connection.search(search_base=entry_dn, + search_filter='(objectclass=*)', + search_scope=BASE, + dereference_aliases=DEREF_NEVER, + attributes=attributes if attributes else self.attributes, + get_operational_attributes=self.get_operational_attributes, + controls=controls) + if not self.connection.strategy.sync: + response, result, request = self.connection.get_response(result, get_request=True) + else: + response = self.connection.response + result = self.connection.result + request = self.connection.request + + if result['result'] in [RESULT_SUCCESS]: + break + sleep(seconds) + counter += 1 + self._store_operation_in_history(request, result, response) + + if len(response) == 1: + return self._create_entry(response[0]) + elif len(response) == 0: + return None + + error_message = 'more than 1 entry returned for a single object search' + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise LDAPCursorError(error_message) + + def new(self, dn): + if log_enabled(BASIC): + log(BASIC, 'creating new entry <%s> for <%s>', dn, self) + dn = safe_dn(dn) + for entry in self.entries: # checks if dn is already used in an cursor entry + if entry.entry_dn == dn: + error_message = 'dn already present in cursor' + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise LDAPCursorError(error_message) + rdns = safe_rdn(dn, decompose=True) + entry = self.entry_class(dn, self) # defines a new empty Entry + for attr in entry.entry_mandatory_attributes: # defines all mandatory attributes as virtual + entry._state.attributes[attr] = self.attribute_class(entry._state.definition[attr], entry, self) + entry.__dict__[attr] = entry._state.attributes[attr] + entry.objectclass.set(self.definition._object_class) + for rdn in rdns: # adds virtual attributes from rdns in entry name (should be more than one with + syntax) + if rdn[0] in entry._state.definition._attributes: + rdn_name = entry._state.definition._attributes[rdn[0]].name # normalize case folding + if rdn_name not in entry._state.attributes: + entry._state.attributes[rdn_name] = self.attribute_class(entry._state.definition[rdn_name], entry, self) + entry.__dict__[rdn_name] = entry._state.attributes[rdn_name] + entry.__dict__[rdn_name].set(rdn[1]) + else: + error_message = 'rdn type \'%s\' not in object class definition' % rdn[0] + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise LDAPCursorError(error_message) + entry._state.set_status(STATUS_VIRTUAL) # set intial status + entry._state.set_status(STATUS_PENDING_CHANGES) # tries to change status to PENDING_CHANGES. If mandatory attributes are missing status is reverted to MANDATORY_MISSING + self.entries.append(entry) + return entry + + def refresh_entry(self, entry, tries=4, seconds=2): + conf_operational_attribute_prefix = get_config_parameter('ABSTRACTION_OPERATIONAL_ATTRIBUTE_PREFIX') + + self._do_not_reset = True + attr_list = [] + if log_enabled(PROTOCOL): + log(PROTOCOL, 'refreshing entry <%s> for <%s>', entry, self) + for attr in entry._state.attributes: # check friendly attribute name in AttrDef, do not check operational attributes + if attr.lower().startswith(conf_operational_attribute_prefix.lower()): + continue + if entry._state.definition[attr].name: + attr_list.append(entry._state.definition[attr].name) + else: + attr_list.append(entry._state.definition[attr].key) + + temp_entry = self._refresh_object(entry.entry_dn, attr_list, tries, seconds=seconds) # if any attributes is added adds only to the entry not to the definition + self._do_not_reset = False + if temp_entry: + temp_entry._state.origin = entry._state.origin + entry.__dict__.clear() + entry.__dict__['_state'] = temp_entry._state + for attr in entry._state.attributes: # returns the attribute key + entry.__dict__[attr] = entry._state.attributes[attr] + + for attr in entry.entry_attributes: # if any attribute of the class was deleted makes it virtual + if attr not in entry._state.attributes and attr in entry.entry_definition._attributes: + entry._state.attributes[attr] = WritableAttribute(entry.entry_definition[attr], entry, self) + entry.__dict__[attr] = entry._state.attributes[attr] + entry._state.set_status(entry._state._initial_status) + return True + return False diff --git a/server/www/packages/packages-linux/x64/ldap3/abstract/entry.py b/server/www/packages/packages-linux/x64/ldap3/abstract/entry.py index 18c0420..ee8bf20 100644 --- a/server/www/packages/packages-linux/x64/ldap3/abstract/entry.py +++ b/server/www/packages/packages-linux/x64/ldap3/abstract/entry.py @@ -1,671 +1,675 @@ -""" -""" - -# Created on 2016.08.19 -# -# Author: Giovanni Cannata -# -# Copyright 2016 - 2018 Giovanni Cannata -# -# This file is part of ldap3. -# -# ldap3 is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# ldap3 is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with ldap3 in the COPYING and COPYING.LESSER files. -# If not, see . - - -import json -try: - from collections import OrderedDict -except ImportError: - from ..utils.ordDict import OrderedDict # for Python 2.6 - -from os import linesep - -from .. import STRING_TYPES, SEQUENCE_TYPES, MODIFY_ADD, MODIFY_REPLACE -from .attribute import WritableAttribute -from .objectDef import ObjectDef -from .attrDef import AttrDef -from ..core.exceptions import LDAPKeyError, LDAPCursorError -from ..utils.conv import check_json_dict, format_json, prepare_for_stream -from ..protocol.rfc2849 import operation_to_ldif, add_ldif_header -from ..utils.dn import safe_dn, safe_rdn, to_dn -from ..utils.repr import to_stdout_encoding -from ..utils.ciDict import CaseInsensitiveWithAliasDict -from ..utils.config import get_config_parameter -from . import STATUS_VIRTUAL, STATUS_WRITABLE, STATUS_PENDING_CHANGES, STATUS_COMMITTED, STATUS_DELETED,\ - STATUS_INIT, STATUS_READY_FOR_DELETION, STATUS_READY_FOR_MOVING, STATUS_READY_FOR_RENAMING, STATUS_MANDATORY_MISSING, STATUSES, INITIAL_STATUSES -from ..core.results import RESULT_SUCCESS -from ..utils.log import log, log_enabled, ERROR, BASIC, PROTOCOL, EXTENDED - - -class EntryState(object): - """Contains data on the status of the entry. Does not pollute the Entry __dict__. - - """ - - def __init__(self, dn, cursor): - self.dn = dn - self._initial_status = None - self._to = None # used for move and rename - self.status = STATUS_INIT - self.attributes = CaseInsensitiveWithAliasDict() - self.raw_attributes = CaseInsensitiveWithAliasDict() - self.response = None - self.cursor = cursor - self.origin = None # reference to the original read-only entry (set when made writable). Needed to update attributes in read-only when modified (only if both refer the same server) - self.read_time = None - self.changes = OrderedDict() # includes changes to commit in a writable entry - if cursor.definition: - self.definition = cursor.definition - else: - self.definition = None - - def __repr__(self): - if self.__dict__ and self.dn is not None: - r = 'DN: ' + to_stdout_encoding(self.dn) + ' - STATUS: ' + ((self._initial_status + ', ') if self._initial_status != self.status else '') + self.status + ' - READ TIME: ' + (self.read_time.isoformat() if self.read_time else '') + linesep - r += 'attributes: ' + ', '.join(sorted(self.attributes.keys())) + linesep - r += 'object def: ' + (', '.join(sorted(self.definition._object_class)) if self.definition._object_class else '') + linesep - r += 'attr defs: ' + ', '.join(sorted(self.definition._attributes.keys())) + linesep - r += 'response: ' + ('present' if self.response else '') + linesep - r += 'cursor: ' + (self.cursor.__class__.__name__ if self.cursor else '') + linesep - return r - else: - return object.__repr__(self) - - def __str__(self): - return self.__repr__() - - def set_status(self, status): - conf_ignored_mandatory_attributes_in_object_def = [v.lower() for v in get_config_parameter('IGNORED_MANDATORY_ATTRIBUTES_IN_OBJECT_DEF')] - if status not in STATUSES: - error_message = 'invalid entry status ' + str(status) - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise LDAPCursorError(error_message) - if status in INITIAL_STATUSES: - self._initial_status = status - self.status = status - if status == STATUS_DELETED: - self._initial_status = STATUS_VIRTUAL - if status == STATUS_COMMITTED: - self._initial_status = STATUS_WRITABLE - if self.status == STATUS_VIRTUAL or (self.status == STATUS_PENDING_CHANGES and self._initial_status == STATUS_VIRTUAL): # checks if all mandatory attributes are present in new entries - for attr in self.definition._attributes: - if self.definition._attributes[attr].mandatory and attr.lower() not in conf_ignored_mandatory_attributes_in_object_def: - if (attr not in self.attributes or self.attributes[attr].virtual) and attr not in self.changes: - self.status = STATUS_MANDATORY_MISSING - break - - -class EntryBase(object): - """The Entry object contains a single LDAP entry. - Attributes can be accessed either by sequence, by assignment - or as dictionary keys. Keys are not case sensitive. - - The Entry object is read only - - - The DN is retrieved by entry_dn - - The cursor reference is in _cursor - - Raw attributes values are retrieved with _raw_attributes and the _raw_attribute() methods - """ - - def __init__(self, dn, cursor): - self.__dict__['_state'] = EntryState(dn, cursor) - - def __repr__(self): - if self.__dict__ and self.entry_dn is not None: - r = 'DN: ' + to_stdout_encoding(self.entry_dn) + ' - STATUS: ' + ((self._state._initial_status + ', ') if self._state._initial_status != self.entry_status else '') + self.entry_status + ' - READ TIME: ' + (self.entry_read_time.isoformat() if self.entry_read_time else '') + linesep - if self._state.attributes: - for attr in sorted(self._state.attributes): - if self._state.attributes[attr] or (hasattr(self._state.attributes[attr], 'changes') and self._state.attributes[attr].changes): - r += ' ' + repr(self._state.attributes[attr]) + linesep - return r - else: - return object.__repr__(self) - - def __str__(self): - return self.__repr__() - - def __iter__(self): - for attribute in self._state.attributes: - yield self._state.attributes[attribute] - # raise StopIteration # deprecated in PEP 479 - return - - def __contains__(self, item): - try: - self.__getitem__(item) - return True - except LDAPKeyError: - return False - - def __getattr__(self, item): - if isinstance(item, STRING_TYPES): - if item == '_state': - return self.__dict__['_state'] - item = ''.join(item.split()).lower() - attr_found = None - for attr in self._state.attributes.keys(): - if item == attr.lower(): - attr_found = attr - break - if not attr_found: - for attr in self._state.attributes.aliases(): - if item == attr.lower(): - attr_found = attr - break - if not attr_found: - for attr in self._state.attributes.keys(): - if item + ';binary' == attr.lower(): - attr_found = attr - break - if not attr_found: - for attr in self._state.attributes.aliases(): - if item + ';binary' == attr.lower(): - attr_found = attr - break - if not attr_found: - for attr in self._state.attributes.keys(): - if item + ';range' in attr.lower(): - attr_found = attr - break - if not attr_found: - for attr in self._state.attributes.aliases(): - if item + ';range' in attr.lower(): - attr_found = attr - break - if not attr_found: - error_message = 'attribute \'%s\' not found' % item - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise LDAPCursorError(error_message) - return self._state.attributes[attr] - error_message = 'attribute name must be a string' - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise LDAPCursorError(error_message) - - def __setattr__(self, item, value): - if item in self._state.attributes: - error_message = 'attribute \'%s\' is read only' % item - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise LDAPCursorError(error_message) - else: - error_message = 'entry is read only, cannot add \'%s\'' % item - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise LDAPCursorError(error_message) - - def __getitem__(self, item): - if isinstance(item, STRING_TYPES): - item = ''.join(item.split()).lower() - attr_found = None - for attr in self._state.attributes.keys(): - if item == attr.lower(): - attr_found = attr - break - if not attr_found: - for attr in self._state.attributes.aliases(): - if item == attr.lower(): - attr_found = attr - break - if not attr_found: - for attr in self._state.attributes.keys(): - if item + ';binary' == attr.lower(): - attr_found = attr - break - if not attr_found: - for attr in self._state.attributes.aliases(): - if item + ';binary' == attr.lower(): - attr_found = attr - break - if not attr_found: - error_message = 'key \'%s\' not found' % item - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise LDAPKeyError(error_message) - return self._state.attributes[attr] - - error_message = 'key must be a string' - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise LDAPKeyError(error_message) - - def __eq__(self, other): - if isinstance(other, EntryBase): - return self.entry_dn == other.entry_dn - - return False - - def __lt__(self, other): - if isinstance(other, EntryBase): - return self.entry_dn <= other.entry_dn - - return False - - @property - def entry_dn(self): - return self._state.dn - - @property - def entry_cursor(self): - return self._state.cursor - - @property - def entry_status(self): - return self._state.status - - @property - def entry_definition(self): - return self._state.definition - - @property - def entry_raw_attributes(self): - return self._state.entry_raw_attributes - - def entry_raw_attribute(self, name): - """ - - :param name: name of the attribute - :return: raw (unencoded) value of the attribute, None if attribute is not found - """ - return self._state.entry_raw_attributes[name] if name in self._state.entry_raw_attributes else None - - @property - def entry_mandatory_attributes(self): - return [attribute for attribute in self.entry_definition._attributes if self.entry_definition._attributes[attribute].mandatory] - - @property - def entry_attributes(self): - return list(self._state.attributes.keys()) - - @property - def entry_attributes_as_dict(self): - return dict((attribute_key, attribute_value.values) for (attribute_key, attribute_value) in self._state.attributes.items()) - - @property - def entry_read_time(self): - return self._state.read_time - - @property - def _changes(self): - return self._state.changes - - def entry_to_json(self, raw=False, indent=4, sort=True, stream=None, checked_attributes=True, include_empty=True): - json_entry = dict() - json_entry['dn'] = self.entry_dn - if checked_attributes: - if not include_empty: - # needed for python 2.6 compatibility - json_entry['attributes'] = dict((key, self.entry_attributes_as_dict[key]) for key in self.entry_attributes_as_dict if self.entry_attributes_as_dict[key]) - else: - json_entry['attributes'] = self.entry_attributes_as_dict - if raw: - if not include_empty: - # needed for python 2.6 compatibility - json_entry['raw'] = dict((key, self.entry_raw_attributes[key]) for key in self.entry_raw_attributes if self.entry_raw_attributes[key]) - else: - json_entry['raw'] = dict(self.entry_raw_attributes) - - if str is bytes: # Python 2 - check_json_dict(json_entry) - - json_output = json.dumps(json_entry, - ensure_ascii=True, - sort_keys=sort, - indent=indent, - check_circular=True, - default=format_json, - separators=(',', ': ')) - - if stream: - stream.write(json_output) - - return json_output - - def entry_to_ldif(self, all_base64=False, line_separator=None, sort_order=None, stream=None): - ldif_lines = operation_to_ldif('searchResponse', [self._state.response], all_base64, sort_order=sort_order) - ldif_lines = add_ldif_header(ldif_lines) - line_separator = line_separator or linesep - ldif_output = line_separator.join(ldif_lines) - if stream: - if stream.tell() == 0: - header = add_ldif_header(['-'])[0] - stream.write(prepare_for_stream(header + line_separator + line_separator)) - stream.write(prepare_for_stream(ldif_output + line_separator + line_separator)) - return ldif_output - - -class Entry(EntryBase): - """The Entry object contains a single LDAP entry. - Attributes can be accessed either by sequence, by assignment - or as dictionary keys. Keys are not case sensitive. - - The Entry object is read only - - - The DN is retrieved by entry_dn - - The Reader reference is in _cursor() - - Raw attributes values are retrieved by the _ra_attributes and - _raw_attribute() methods - - """ - def entry_writable(self, object_def=None, writer_cursor=None, attributes=None, custom_validator=None, auxiliary_class=None): - if not self.entry_cursor.schema: - error_message = 'schema must be available to make an entry writable' - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise LDAPCursorError(error_message) - # returns a new WritableEntry and its Writer cursor - if object_def is None: - if self.entry_cursor.definition._object_class: - object_def = self.entry_definition._object_class - auxiliary_class = self.entry_definition._auxiliary_class + (auxiliary_class if isinstance(auxiliary_class, SEQUENCE_TYPES) else []) - elif 'objectclass' in self: - object_def = self.objectclass.values - - if not object_def: - error_message = 'object class must be specified to make an entry writable' - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise LDAPCursorError(error_message) - - if not isinstance(object_def, ObjectDef): - object_def = ObjectDef(object_def, self.entry_cursor.schema, custom_validator, auxiliary_class) - - if attributes: - if isinstance(attributes, STRING_TYPES): - attributes = [attributes] - - if isinstance(attributes, SEQUENCE_TYPES): - for attribute in attributes: - if attribute not in object_def._attributes: - error_message = 'attribute \'%s\' not in schema for \'%s\'' % (attribute, object_def) - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise LDAPCursorError(error_message) - else: - attributes = [] - - if not writer_cursor: - from .cursor import Writer # local import to avoid circular reference in import at startup - writable_cursor = Writer(self.entry_cursor.connection, object_def) - else: - writable_cursor = writer_cursor - - if attributes: # force reading of attributes - writable_entry = writable_cursor._refresh_object(self.entry_dn, list(attributes) + self.entry_attributes) - else: - writable_entry = writable_cursor._create_entry(self._state.response) - writable_cursor.entries.append(writable_entry) - writable_entry._state.read_time = self.entry_read_time - writable_entry._state.origin = self # reference to the original read-only entry - # checks original entry for custom definitions in AttrDefs - for attr in writable_entry._state.origin.entry_definition._attributes: - original_attr = writable_entry._state.origin.entry_definition._attributes[attr] - if attr != original_attr.name and attr not in writable_entry._state.attributes: - old_attr_def = writable_entry.entry_definition._attributes[original_attr.name] - new_attr_def = AttrDef(original_attr.name, - key=attr, - validate=original_attr.validate, - pre_query=original_attr.pre_query, - post_query=original_attr.post_query, - default=original_attr.default, - dereference_dn=original_attr.dereference_dn, - description=original_attr.description, - mandatory=old_attr_def.mandatory, # keeps value read from schema - single_value=old_attr_def.single_value, # keeps value read from schema - alias=original_attr.other_names) - object_def = writable_entry.entry_definition - object_def -= old_attr_def - object_def += new_attr_def - # updates attribute name in entry attributes - new_attr = WritableAttribute(new_attr_def, writable_entry, writable_cursor) - if original_attr.name in writable_entry._state.attributes: - new_attr.other_names = writable_entry._state.attributes[original_attr.name].other_names - new_attr.raw_values = writable_entry._state.attributes[original_attr.name].raw_values - new_attr.values = writable_entry._state.attributes[original_attr.name].values - new_attr.response = writable_entry._state.attributes[original_attr.name].response - writable_entry._state.attributes[attr] = new_attr - # writable_entry._state.attributes.set_alias(attr, new_attr.other_names) - del writable_entry._state.attributes[original_attr.name] - - writable_entry._state.set_status(STATUS_WRITABLE) - return writable_entry - - -class WritableEntry(EntryBase): - def __setitem__(self, key, value): - if value is not Ellipsis: # hack for using implicit operators in writable attributes - self.__setattr__(key, value) - - def __setattr__(self, item, value): - conf_attributes_excluded_from_object_def = [v.lower() for v in get_config_parameter('ATTRIBUTES_EXCLUDED_FROM_OBJECT_DEF')] - if item == '_state' and isinstance(value, EntryState): - self.__dict__['_state'] = value - return - - if value is not Ellipsis: # hack for using implicit operators in writable attributes - # checks if using an alias - if item in self.entry_cursor.definition._attributes or item.lower() in conf_attributes_excluded_from_object_def: - if item not in self._state.attributes: # setting value to an attribute still without values - new_attribute = WritableAttribute(self.entry_cursor.definition._attributes[item], self, cursor=self.entry_cursor) - self._state.attributes[str(item)] = new_attribute # force item to a string for key in attributes dict - self._state.attributes[item].set(value) # try to add to new_values - else: - error_message = 'attribute \'%s\' not defined' % item - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise LDAPCursorError(error_message) - - def __getattr__(self, item): - if isinstance(item, STRING_TYPES): - if item == '_state': - return self.__dict__['_state'] - item = ''.join(item.split()).lower() - for attr in self._state.attributes.keys(): - if item == attr.lower(): - return self._state.attributes[attr] - for attr in self._state.attributes.aliases(): - if item == attr.lower(): - return self._state.attributes[attr] - if item in self.entry_definition._attributes: # item is a new attribute to commit, creates the AttrDef and add to the attributes to retrive - self._state.attributes[item] = WritableAttribute(self.entry_definition._attributes[item], self, self.entry_cursor) - self.entry_cursor.attributes.add(item) - return self._state.attributes[item] - error_message = 'attribute \'%s\' not defined' % item - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise LDAPCursorError(error_message) - else: - error_message = 'attribute name must be a string' - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise LDAPCursorError(error_message) - - @property - def entry_virtual_attributes(self): - return [attr for attr in self.entry_attributes if self[attr].virtual] - - def entry_commit_changes(self, refresh=True, controls=None, clear_history=True): - if clear_history: - self.entry_cursor._reset_history() - - if self.entry_status == STATUS_READY_FOR_DELETION: - result = self.entry_cursor.connection.delete(self.entry_dn, controls) - if not self.entry_cursor.connection.strategy.sync: - response, result, request = self.entry_cursor.connection.get_response(result, get_request=True) - else: - response = self.entry_cursor.connection.response - result = self.entry_cursor.connection.result - request = self.entry_cursor.connection.request - self.entry_cursor._store_operation_in_history(request, result, response) - if result['result'] == RESULT_SUCCESS: - dn = self.entry_dn - if self._state.origin and self.entry_cursor.connection.server == self._state.origin.entry_cursor.connection.server: # deletes original read-only Entry - cursor = self._state.origin.entry_cursor - self._state.origin.__dict__.clear() - self._state.origin.__dict__['_state'] = EntryState(dn, cursor) - self._state.origin._state.set_status(STATUS_DELETED) - cursor = self.entry_cursor - self.__dict__.clear() - self._state = EntryState(dn, cursor) - self._state.set_status(STATUS_DELETED) - return True - return False - elif self.entry_status == STATUS_READY_FOR_MOVING: - result = self.entry_cursor.connection.modify_dn(self.entry_dn, '+'.join(safe_rdn(self.entry_dn)), new_superior=self._state._to) - if not self.entry_cursor.connection.strategy.sync: - response, result, request = self.entry_cursor.connection.get_response(result, get_request=True) - else: - response = self.entry_cursor.connection.response - result = self.entry_cursor.connection.result - request = self.entry_cursor.connection.request - self.entry_cursor._store_operation_in_history(request, result, response) - if result['result'] == RESULT_SUCCESS: - self._state.dn = safe_dn('+'.join(safe_rdn(self.entry_dn)) + ',' + self._state._to) - if refresh: - if self.entry_refresh(): - if self._state.origin and self.entry_cursor.connection.server == self._state.origin.entry_cursor.connection.server: # refresh dn of origin - self._state.origin._state.dn = self.entry_dn - self._state.set_status(STATUS_COMMITTED) - self._state._to = None - return True - return False - elif self.entry_status == STATUS_READY_FOR_RENAMING: - rdn = '+'.join(safe_rdn(self._state._to)) - result = self.entry_cursor.connection.modify_dn(self.entry_dn, rdn) - if not self.entry_cursor.connection.strategy.sync: - response, result, request = self.entry_cursor.connection.get_response(result, get_request=True) - else: - response = self.entry_cursor.connection.response - result = self.entry_cursor.connection.result - request = self.entry_cursor.connection.request - self.entry_cursor._store_operation_in_history(request, result, response) - if result['result'] == RESULT_SUCCESS: - self._state.dn = rdn + ',' + ','.join(to_dn(self.entry_dn)[1:]) - if refresh: - if self.entry_refresh(): - if self._state.origin and self.entry_cursor.connection.server == self._state.origin.entry_cursor.connection.server: # refresh dn of origin - self._state.origin._state.dn = self.entry_dn - self._state.set_status(STATUS_COMMITTED) - self._state._to = None - return True - return False - elif self.entry_status in [STATUS_VIRTUAL, STATUS_MANDATORY_MISSING]: - missing_attributes = [] - for attr in self.entry_mandatory_attributes: - if (attr not in self._state.attributes or self._state.attributes[attr].virtual) and attr not in self._changes: - missing_attributes.append('\'' + attr + '\'') - error_message = 'mandatory attributes %s missing in entry %s' % (', '.join(missing_attributes), self.entry_dn) - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise LDAPCursorError(error_message) - elif self.entry_status == STATUS_PENDING_CHANGES: - if self._changes: - if self.entry_definition._auxiliary_class: # checks if an attribute is from an auxiliary class and adds it to the objectClass attribute if not present - for attr in self._changes: - # checks schema to see if attribute is defined in one of the already present object classes - attr_classes = self.entry_cursor.schema.attribute_types[attr].mandatory_in + self.entry_cursor.schema.attribute_types[attr].optional_in - for object_class in self.objectclass: - if object_class in attr_classes: - break - else: # executed only if the attribute class is not present in the objectClass attribute - # checks if attribute is defined in one of the possible auxiliary classes - for aux_class in self.entry_definition._auxiliary_class: - if aux_class in attr_classes: - if self._state._initial_status == STATUS_VIRTUAL: # entry is new, there must be a pending objectClass MODIFY_REPLACE - self._changes['objectClass'][0][1].append(aux_class) - else: - self.objectclass += aux_class - if self._state._initial_status == STATUS_VIRTUAL: - new_attributes = dict() - for attr in self._changes: - new_attributes[attr] = self._changes[attr][0][1] - result = self.entry_cursor.connection.add(self.entry_dn, None, new_attributes, controls) - else: - result = self.entry_cursor.connection.modify(self.entry_dn, self._changes, controls) - - if not self.entry_cursor.connection.strategy.sync: # asynchronous request - response, result, request = self.entry_cursor.connection.get_response(result, get_request=True) - else: - response = self.entry_cursor.connection.response - result = self.entry_cursor.connection.result - request = self.entry_cursor.connection.request - self.entry_cursor._store_operation_in_history(request, result, response) - - if result['result'] == RESULT_SUCCESS: - if refresh: - if self.entry_refresh(): - if self._state.origin and self.entry_cursor.connection.server == self._state.origin.entry_cursor.connection.server: # updates original read-only entry if present - for attr in self: # adds AttrDefs from writable entry to origin entry definition if some is missing - if attr.key in self.entry_definition._attributes and attr.key not in self._state.origin.entry_definition._attributes: - self._state.origin.entry_cursor.definition.add_attribute(self.entry_cursor.definition._attributes[attr.key]) # adds AttrDef from writable entry to original entry if missing - temp_entry = self._state.origin.entry_cursor._create_entry(self._state.response) - self._state.origin.__dict__.clear() - self._state.origin.__dict__['_state'] = temp_entry._state - for attr in self: # returns the whole attribute object - if not attr.virtual: - self._state.origin.__dict__[attr.key] = self._state.origin._state.attributes[attr.key] - self._state.origin._state.read_time = self.entry_read_time - else: - self.entry_discard_changes() # if not refreshed remove committed changes - self._state.set_status(STATUS_COMMITTED) - return True - return False - - def entry_discard_changes(self): - self._changes.clear() - self._state.set_status(self._state._initial_status) - - def entry_delete(self): - if self.entry_status not in [STATUS_WRITABLE, STATUS_COMMITTED, STATUS_READY_FOR_DELETION]: - error_message = 'cannot delete entry, invalid status: ' + self.entry_status - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise LDAPCursorError(error_message) - self._state.set_status(STATUS_READY_FOR_DELETION) - - def entry_refresh(self, tries=4, seconds=2): - """ - - Refreshes the entry from the LDAP Server - """ - if self.entry_cursor.connection: - if self.entry_cursor.refresh_entry(self, tries, seconds): - return True - - return False - - def entry_move(self, destination_dn): - if self.entry_status not in [STATUS_WRITABLE, STATUS_COMMITTED, STATUS_READY_FOR_MOVING]: - error_message = 'cannot move entry, invalid status: ' + self.entry_status - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise LDAPCursorError(error_message) - self._state._to = safe_dn(destination_dn) - self._state.set_status(STATUS_READY_FOR_MOVING) - - def entry_rename(self, new_name): - if self.entry_status not in [STATUS_WRITABLE, STATUS_COMMITTED, STATUS_READY_FOR_RENAMING]: - error_message = 'cannot rename entry, invalid status: ' + self.entry_status - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', error_message, self) - raise LDAPCursorError(error_message) - self._state._to = new_name - self._state.set_status(STATUS_READY_FOR_RENAMING) - - @property - def entry_changes(self): - return self._changes +""" +""" + +# Created on 2016.08.19 +# +# Author: Giovanni Cannata +# +# Copyright 2016 - 2018 Giovanni Cannata +# +# This file is part of ldap3. +# +# ldap3 is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ldap3 is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with ldap3 in the COPYING and COPYING.LESSER files. +# If not, see . + + +import json +try: + from collections import OrderedDict +except ImportError: + from ..utils.ordDict import OrderedDict # for Python 2.6 + +from os import linesep + +from .. import STRING_TYPES, SEQUENCE_TYPES, MODIFY_ADD, MODIFY_REPLACE +from .attribute import WritableAttribute +from .objectDef import ObjectDef +from .attrDef import AttrDef +from ..core.exceptions import LDAPKeyError, LDAPCursorError +from ..utils.conv import check_json_dict, format_json, prepare_for_stream +from ..protocol.rfc2849 import operation_to_ldif, add_ldif_header +from ..utils.dn import safe_dn, safe_rdn, to_dn +from ..utils.repr import to_stdout_encoding +from ..utils.ciDict import CaseInsensitiveWithAliasDict +from ..utils.config import get_config_parameter +from . import STATUS_VIRTUAL, STATUS_WRITABLE, STATUS_PENDING_CHANGES, STATUS_COMMITTED, STATUS_DELETED,\ + STATUS_INIT, STATUS_READY_FOR_DELETION, STATUS_READY_FOR_MOVING, STATUS_READY_FOR_RENAMING, STATUS_MANDATORY_MISSING, STATUSES, INITIAL_STATUSES +from ..core.results import RESULT_SUCCESS +from ..utils.log import log, log_enabled, ERROR, BASIC, PROTOCOL, EXTENDED + + +class EntryState(object): + """Contains data on the status of the entry. Does not pollute the Entry __dict__. + + """ + + def __init__(self, dn, cursor): + self.dn = dn + self._initial_status = None + self._to = None # used for move and rename + self.status = STATUS_INIT + self.attributes = CaseInsensitiveWithAliasDict() + self.raw_attributes = CaseInsensitiveWithAliasDict() + self.response = None + self.cursor = cursor + self.origin = None # reference to the original read-only entry (set when made writable). Needed to update attributes in read-only when modified (only if both refer the same server) + self.read_time = None + self.changes = OrderedDict() # includes changes to commit in a writable entry + if cursor.definition: + self.definition = cursor.definition + else: + self.definition = None + + def __repr__(self): + if self.__dict__ and self.dn is not None: + r = 'DN: ' + to_stdout_encoding(self.dn) + ' - STATUS: ' + ((self._initial_status + ', ') if self._initial_status != self.status else '') + self.status + ' - READ TIME: ' + (self.read_time.isoformat() if self.read_time else '') + linesep + r += 'attributes: ' + ', '.join(sorted(self.attributes.keys())) + linesep + r += 'object def: ' + (', '.join(sorted(self.definition._object_class)) if self.definition._object_class else '') + linesep + r += 'attr defs: ' + ', '.join(sorted(self.definition._attributes.keys())) + linesep + r += 'response: ' + ('present' if self.response else '') + linesep + r += 'cursor: ' + (self.cursor.__class__.__name__ if self.cursor else '') + linesep + return r + else: + return object.__repr__(self) + + def __str__(self): + return self.__repr__() + + def set_status(self, status): + conf_ignored_mandatory_attributes_in_object_def = [v.lower() for v in get_config_parameter('IGNORED_MANDATORY_ATTRIBUTES_IN_OBJECT_DEF')] + if status not in STATUSES: + error_message = 'invalid entry status ' + str(status) + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise LDAPCursorError(error_message) + if status in INITIAL_STATUSES: + self._initial_status = status + self.status = status + if status == STATUS_DELETED: + self._initial_status = STATUS_VIRTUAL + if status == STATUS_COMMITTED: + self._initial_status = STATUS_WRITABLE + if self.status == STATUS_VIRTUAL or (self.status == STATUS_PENDING_CHANGES and self._initial_status == STATUS_VIRTUAL): # checks if all mandatory attributes are present in new entries + for attr in self.definition._attributes: + if self.definition._attributes[attr].mandatory and attr.lower() not in conf_ignored_mandatory_attributes_in_object_def: + if (attr not in self.attributes or self.attributes[attr].virtual) and attr not in self.changes: + self.status = STATUS_MANDATORY_MISSING + break + + @property + def entry_raw_attributes(self): + return self.raw_attributes + + +class EntryBase(object): + """The Entry object contains a single LDAP entry. + Attributes can be accessed either by sequence, by assignment + or as dictionary keys. Keys are not case sensitive. + + The Entry object is read only + + - The DN is retrieved by entry_dn + - The cursor reference is in _cursor + - Raw attributes values are retrieved with _raw_attributes and the _raw_attribute() methods + """ + + def __init__(self, dn, cursor): + self.__dict__['_state'] = EntryState(dn, cursor) + + def __repr__(self): + if self.__dict__ and self.entry_dn is not None: + r = 'DN: ' + to_stdout_encoding(self.entry_dn) + ' - STATUS: ' + ((self._state._initial_status + ', ') if self._state._initial_status != self.entry_status else '') + self.entry_status + ' - READ TIME: ' + (self.entry_read_time.isoformat() if self.entry_read_time else '') + linesep + if self._state.attributes: + for attr in sorted(self._state.attributes): + if self._state.attributes[attr] or (hasattr(self._state.attributes[attr], 'changes') and self._state.attributes[attr].changes): + r += ' ' + repr(self._state.attributes[attr]) + linesep + return r + else: + return object.__repr__(self) + + def __str__(self): + return self.__repr__() + + def __iter__(self): + for attribute in self._state.attributes: + yield self._state.attributes[attribute] + # raise StopIteration # deprecated in PEP 479 + return + + def __contains__(self, item): + try: + self.__getitem__(item) + return True + except LDAPKeyError: + return False + + def __getattr__(self, item): + if isinstance(item, STRING_TYPES): + if item == '_state': + return self.__dict__['_state'] + item = ''.join(item.split()).lower() + attr_found = None + for attr in self._state.attributes.keys(): + if item == attr.lower(): + attr_found = attr + break + if not attr_found: + for attr in self._state.attributes.aliases(): + if item == attr.lower(): + attr_found = attr + break + if not attr_found: + for attr in self._state.attributes.keys(): + if item + ';binary' == attr.lower(): + attr_found = attr + break + if not attr_found: + for attr in self._state.attributes.aliases(): + if item + ';binary' == attr.lower(): + attr_found = attr + break + if not attr_found: + for attr in self._state.attributes.keys(): + if item + ';range' in attr.lower(): + attr_found = attr + break + if not attr_found: + for attr in self._state.attributes.aliases(): + if item + ';range' in attr.lower(): + attr_found = attr + break + if not attr_found: + error_message = 'attribute \'%s\' not found' % item + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise LDAPCursorError(error_message) + return self._state.attributes[attr] + error_message = 'attribute name must be a string' + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise LDAPCursorError(error_message) + + def __setattr__(self, item, value): + if item in self._state.attributes: + error_message = 'attribute \'%s\' is read only' % item + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise LDAPCursorError(error_message) + else: + error_message = 'entry is read only, cannot add \'%s\'' % item + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise LDAPCursorError(error_message) + + def __getitem__(self, item): + if isinstance(item, STRING_TYPES): + item = ''.join(item.split()).lower() + attr_found = None + for attr in self._state.attributes.keys(): + if item == attr.lower(): + attr_found = attr + break + if not attr_found: + for attr in self._state.attributes.aliases(): + if item == attr.lower(): + attr_found = attr + break + if not attr_found: + for attr in self._state.attributes.keys(): + if item + ';binary' == attr.lower(): + attr_found = attr + break + if not attr_found: + for attr in self._state.attributes.aliases(): + if item + ';binary' == attr.lower(): + attr_found = attr + break + if not attr_found: + error_message = 'key \'%s\' not found' % item + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise LDAPKeyError(error_message) + return self._state.attributes[attr] + + error_message = 'key must be a string' + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise LDAPKeyError(error_message) + + def __eq__(self, other): + if isinstance(other, EntryBase): + return self.entry_dn == other.entry_dn + + return False + + def __lt__(self, other): + if isinstance(other, EntryBase): + return self.entry_dn <= other.entry_dn + + return False + + @property + def entry_dn(self): + return self._state.dn + + @property + def entry_cursor(self): + return self._state.cursor + + @property + def entry_status(self): + return self._state.status + + @property + def entry_definition(self): + return self._state.definition + + @property + def entry_raw_attributes(self): + return self._state.raw_attributes + + def entry_raw_attribute(self, name): + """ + + :param name: name of the attribute + :return: raw (unencoded) value of the attribute, None if attribute is not found + """ + return self._state.raw_attributes[name] if name in self._state.raw_attributes else None + + @property + def entry_mandatory_attributes(self): + return [attribute for attribute in self.entry_definition._attributes if self.entry_definition._attributes[attribute].mandatory] + + @property + def entry_attributes(self): + return list(self._state.attributes.keys()) + + @property + def entry_attributes_as_dict(self): + return dict((attribute_key, attribute_value.values) for (attribute_key, attribute_value) in self._state.attributes.items()) + + @property + def entry_read_time(self): + return self._state.read_time + + @property + def _changes(self): + return self._state.changes + + def entry_to_json(self, raw=False, indent=4, sort=True, stream=None, checked_attributes=True, include_empty=True): + json_entry = dict() + json_entry['dn'] = self.entry_dn + if checked_attributes: + if not include_empty: + # needed for python 2.6 compatibility + json_entry['attributes'] = dict((key, self.entry_attributes_as_dict[key]) for key in self.entry_attributes_as_dict if self.entry_attributes_as_dict[key]) + else: + json_entry['attributes'] = self.entry_attributes_as_dict + if raw: + if not include_empty: + # needed for python 2.6 compatibility + json_entry['raw'] = dict((key, self.entry_raw_attributes[key]) for key in self.entry_raw_attributes if self.entry_raw_attributes[key]) + else: + json_entry['raw'] = dict(self.entry_raw_attributes) + + if str is bytes: # Python 2 + check_json_dict(json_entry) + + json_output = json.dumps(json_entry, + ensure_ascii=True, + sort_keys=sort, + indent=indent, + check_circular=True, + default=format_json, + separators=(',', ': ')) + + if stream: + stream.write(json_output) + + return json_output + + def entry_to_ldif(self, all_base64=False, line_separator=None, sort_order=None, stream=None): + ldif_lines = operation_to_ldif('searchResponse', [self._state.response], all_base64, sort_order=sort_order) + ldif_lines = add_ldif_header(ldif_lines) + line_separator = line_separator or linesep + ldif_output = line_separator.join(ldif_lines) + if stream: + if stream.tell() == 0: + header = add_ldif_header(['-'])[0] + stream.write(prepare_for_stream(header + line_separator + line_separator)) + stream.write(prepare_for_stream(ldif_output + line_separator + line_separator)) + return ldif_output + + +class Entry(EntryBase): + """The Entry object contains a single LDAP entry. + Attributes can be accessed either by sequence, by assignment + or as dictionary keys. Keys are not case sensitive. + + The Entry object is read only + + - The DN is retrieved by entry_dn + - The Reader reference is in _cursor() + - Raw attributes values are retrieved by the _ra_attributes and + _raw_attribute() methods + + """ + def entry_writable(self, object_def=None, writer_cursor=None, attributes=None, custom_validator=None, auxiliary_class=None): + if not self.entry_cursor.schema: + error_message = 'schema must be available to make an entry writable' + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise LDAPCursorError(error_message) + # returns a new WritableEntry and its Writer cursor + if object_def is None: + if self.entry_cursor.definition._object_class: + object_def = self.entry_definition._object_class + auxiliary_class = self.entry_definition._auxiliary_class + (auxiliary_class if isinstance(auxiliary_class, SEQUENCE_TYPES) else []) + elif 'objectclass' in self: + object_def = self.objectclass.values + + if not object_def: + error_message = 'object class must be specified to make an entry writable' + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise LDAPCursorError(error_message) + + if not isinstance(object_def, ObjectDef): + object_def = ObjectDef(object_def, self.entry_cursor.schema, custom_validator, auxiliary_class) + + if attributes: + if isinstance(attributes, STRING_TYPES): + attributes = [attributes] + + if isinstance(attributes, SEQUENCE_TYPES): + for attribute in attributes: + if attribute not in object_def._attributes: + error_message = 'attribute \'%s\' not in schema for \'%s\'' % (attribute, object_def) + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise LDAPCursorError(error_message) + else: + attributes = [] + + if not writer_cursor: + from .cursor import Writer # local import to avoid circular reference in import at startup + writable_cursor = Writer(self.entry_cursor.connection, object_def) + else: + writable_cursor = writer_cursor + + if attributes: # force reading of attributes + writable_entry = writable_cursor._refresh_object(self.entry_dn, list(attributes) + self.entry_attributes) + else: + writable_entry = writable_cursor._create_entry(self._state.response) + writable_cursor.entries.append(writable_entry) + writable_entry._state.read_time = self.entry_read_time + writable_entry._state.origin = self # reference to the original read-only entry + # checks original entry for custom definitions in AttrDefs + for attr in writable_entry._state.origin.entry_definition._attributes: + original_attr = writable_entry._state.origin.entry_definition._attributes[attr] + if attr != original_attr.name and attr not in writable_entry._state.attributes: + old_attr_def = writable_entry.entry_definition._attributes[original_attr.name] + new_attr_def = AttrDef(original_attr.name, + key=attr, + validate=original_attr.validate, + pre_query=original_attr.pre_query, + post_query=original_attr.post_query, + default=original_attr.default, + dereference_dn=original_attr.dereference_dn, + description=original_attr.description, + mandatory=old_attr_def.mandatory, # keeps value read from schema + single_value=old_attr_def.single_value, # keeps value read from schema + alias=original_attr.other_names) + object_def = writable_entry.entry_definition + object_def -= old_attr_def + object_def += new_attr_def + # updates attribute name in entry attributes + new_attr = WritableAttribute(new_attr_def, writable_entry, writable_cursor) + if original_attr.name in writable_entry._state.attributes: + new_attr.other_names = writable_entry._state.attributes[original_attr.name].other_names + new_attr.raw_values = writable_entry._state.attributes[original_attr.name].raw_values + new_attr.values = writable_entry._state.attributes[original_attr.name].values + new_attr.response = writable_entry._state.attributes[original_attr.name].response + writable_entry._state.attributes[attr] = new_attr + # writable_entry._state.attributes.set_alias(attr, new_attr.other_names) + del writable_entry._state.attributes[original_attr.name] + + writable_entry._state.set_status(STATUS_WRITABLE) + return writable_entry + + +class WritableEntry(EntryBase): + def __setitem__(self, key, value): + if value is not Ellipsis: # hack for using implicit operators in writable attributes + self.__setattr__(key, value) + + def __setattr__(self, item, value): + conf_attributes_excluded_from_object_def = [v.lower() for v in get_config_parameter('ATTRIBUTES_EXCLUDED_FROM_OBJECT_DEF')] + if item == '_state' and isinstance(value, EntryState): + self.__dict__['_state'] = value + return + + if value is not Ellipsis: # hack for using implicit operators in writable attributes + # checks if using an alias + if item in self.entry_cursor.definition._attributes or item.lower() in conf_attributes_excluded_from_object_def: + if item not in self._state.attributes: # setting value to an attribute still without values + new_attribute = WritableAttribute(self.entry_cursor.definition._attributes[item], self, cursor=self.entry_cursor) + self._state.attributes[str(item)] = new_attribute # force item to a string for key in attributes dict + self._state.attributes[item].set(value) # try to add to new_values + else: + error_message = 'attribute \'%s\' not defined' % item + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise LDAPCursorError(error_message) + + def __getattr__(self, item): + if isinstance(item, STRING_TYPES): + if item == '_state': + return self.__dict__['_state'] + item = ''.join(item.split()).lower() + for attr in self._state.attributes.keys(): + if item == attr.lower(): + return self._state.attributes[attr] + for attr in self._state.attributes.aliases(): + if item == attr.lower(): + return self._state.attributes[attr] + if item in self.entry_definition._attributes: # item is a new attribute to commit, creates the AttrDef and add to the attributes to retrive + self._state.attributes[item] = WritableAttribute(self.entry_definition._attributes[item], self, self.entry_cursor) + self.entry_cursor.attributes.add(item) + return self._state.attributes[item] + error_message = 'attribute \'%s\' not defined' % item + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise LDAPCursorError(error_message) + else: + error_message = 'attribute name must be a string' + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise LDAPCursorError(error_message) + + @property + def entry_virtual_attributes(self): + return [attr for attr in self.entry_attributes if self[attr].virtual] + + def entry_commit_changes(self, refresh=True, controls=None, clear_history=True): + if clear_history: + self.entry_cursor._reset_history() + + if self.entry_status == STATUS_READY_FOR_DELETION: + result = self.entry_cursor.connection.delete(self.entry_dn, controls) + if not self.entry_cursor.connection.strategy.sync: + response, result, request = self.entry_cursor.connection.get_response(result, get_request=True) + else: + response = self.entry_cursor.connection.response + result = self.entry_cursor.connection.result + request = self.entry_cursor.connection.request + self.entry_cursor._store_operation_in_history(request, result, response) + if result['result'] == RESULT_SUCCESS: + dn = self.entry_dn + if self._state.origin and self.entry_cursor.connection.server == self._state.origin.entry_cursor.connection.server: # deletes original read-only Entry + cursor = self._state.origin.entry_cursor + self._state.origin.__dict__.clear() + self._state.origin.__dict__['_state'] = EntryState(dn, cursor) + self._state.origin._state.set_status(STATUS_DELETED) + cursor = self.entry_cursor + self.__dict__.clear() + self._state = EntryState(dn, cursor) + self._state.set_status(STATUS_DELETED) + return True + return False + elif self.entry_status == STATUS_READY_FOR_MOVING: + result = self.entry_cursor.connection.modify_dn(self.entry_dn, '+'.join(safe_rdn(self.entry_dn)), new_superior=self._state._to) + if not self.entry_cursor.connection.strategy.sync: + response, result, request = self.entry_cursor.connection.get_response(result, get_request=True) + else: + response = self.entry_cursor.connection.response + result = self.entry_cursor.connection.result + request = self.entry_cursor.connection.request + self.entry_cursor._store_operation_in_history(request, result, response) + if result['result'] == RESULT_SUCCESS: + self._state.dn = safe_dn('+'.join(safe_rdn(self.entry_dn)) + ',' + self._state._to) + if refresh: + if self.entry_refresh(): + if self._state.origin and self.entry_cursor.connection.server == self._state.origin.entry_cursor.connection.server: # refresh dn of origin + self._state.origin._state.dn = self.entry_dn + self._state.set_status(STATUS_COMMITTED) + self._state._to = None + return True + return False + elif self.entry_status == STATUS_READY_FOR_RENAMING: + rdn = '+'.join(safe_rdn(self._state._to)) + result = self.entry_cursor.connection.modify_dn(self.entry_dn, rdn) + if not self.entry_cursor.connection.strategy.sync: + response, result, request = self.entry_cursor.connection.get_response(result, get_request=True) + else: + response = self.entry_cursor.connection.response + result = self.entry_cursor.connection.result + request = self.entry_cursor.connection.request + self.entry_cursor._store_operation_in_history(request, result, response) + if result['result'] == RESULT_SUCCESS: + self._state.dn = rdn + ',' + ','.join(to_dn(self.entry_dn)[1:]) + if refresh: + if self.entry_refresh(): + if self._state.origin and self.entry_cursor.connection.server == self._state.origin.entry_cursor.connection.server: # refresh dn of origin + self._state.origin._state.dn = self.entry_dn + self._state.set_status(STATUS_COMMITTED) + self._state._to = None + return True + return False + elif self.entry_status in [STATUS_VIRTUAL, STATUS_MANDATORY_MISSING]: + missing_attributes = [] + for attr in self.entry_mandatory_attributes: + if (attr not in self._state.attributes or self._state.attributes[attr].virtual) and attr not in self._changes: + missing_attributes.append('\'' + attr + '\'') + error_message = 'mandatory attributes %s missing in entry %s' % (', '.join(missing_attributes), self.entry_dn) + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise LDAPCursorError(error_message) + elif self.entry_status == STATUS_PENDING_CHANGES: + if self._changes: + if self.entry_definition._auxiliary_class: # checks if an attribute is from an auxiliary class and adds it to the objectClass attribute if not present + for attr in self._changes: + # checks schema to see if attribute is defined in one of the already present object classes + attr_classes = self.entry_cursor.schema.attribute_types[attr].mandatory_in + self.entry_cursor.schema.attribute_types[attr].optional_in + for object_class in self.objectclass: + if object_class in attr_classes: + break + else: # executed only if the attribute class is not present in the objectClass attribute + # checks if attribute is defined in one of the possible auxiliary classes + for aux_class in self.entry_definition._auxiliary_class: + if aux_class in attr_classes: + if self._state._initial_status == STATUS_VIRTUAL: # entry is new, there must be a pending objectClass MODIFY_REPLACE + self._changes['objectClass'][0][1].append(aux_class) + else: + self.objectclass += aux_class + if self._state._initial_status == STATUS_VIRTUAL: + new_attributes = dict() + for attr in self._changes: + new_attributes[attr] = self._changes[attr][0][1] + result = self.entry_cursor.connection.add(self.entry_dn, None, new_attributes, controls) + else: + result = self.entry_cursor.connection.modify(self.entry_dn, self._changes, controls) + + if not self.entry_cursor.connection.strategy.sync: # asynchronous request + response, result, request = self.entry_cursor.connection.get_response(result, get_request=True) + else: + response = self.entry_cursor.connection.response + result = self.entry_cursor.connection.result + request = self.entry_cursor.connection.request + self.entry_cursor._store_operation_in_history(request, result, response) + + if result['result'] == RESULT_SUCCESS: + if refresh: + if self.entry_refresh(): + if self._state.origin and self.entry_cursor.connection.server == self._state.origin.entry_cursor.connection.server: # updates original read-only entry if present + for attr in self: # adds AttrDefs from writable entry to origin entry definition if some is missing + if attr.key in self.entry_definition._attributes and attr.key not in self._state.origin.entry_definition._attributes: + self._state.origin.entry_cursor.definition.add_attribute(self.entry_cursor.definition._attributes[attr.key]) # adds AttrDef from writable entry to original entry if missing + temp_entry = self._state.origin.entry_cursor._create_entry(self._state.response) + self._state.origin.__dict__.clear() + self._state.origin.__dict__['_state'] = temp_entry._state + for attr in self: # returns the whole attribute object + if not attr.virtual: + self._state.origin.__dict__[attr.key] = self._state.origin._state.attributes[attr.key] + self._state.origin._state.read_time = self.entry_read_time + else: + self.entry_discard_changes() # if not refreshed remove committed changes + self._state.set_status(STATUS_COMMITTED) + return True + return False + + def entry_discard_changes(self): + self._changes.clear() + self._state.set_status(self._state._initial_status) + + def entry_delete(self): + if self.entry_status not in [STATUS_WRITABLE, STATUS_COMMITTED, STATUS_READY_FOR_DELETION]: + error_message = 'cannot delete entry, invalid status: ' + self.entry_status + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise LDAPCursorError(error_message) + self._state.set_status(STATUS_READY_FOR_DELETION) + + def entry_refresh(self, tries=4, seconds=2): + """ + + Refreshes the entry from the LDAP Server + """ + if self.entry_cursor.connection: + if self.entry_cursor.refresh_entry(self, tries, seconds): + return True + + return False + + def entry_move(self, destination_dn): + if self.entry_status not in [STATUS_WRITABLE, STATUS_COMMITTED, STATUS_READY_FOR_MOVING]: + error_message = 'cannot move entry, invalid status: ' + self.entry_status + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise LDAPCursorError(error_message) + self._state._to = safe_dn(destination_dn) + self._state.set_status(STATUS_READY_FOR_MOVING) + + def entry_rename(self, new_name): + if self.entry_status not in [STATUS_WRITABLE, STATUS_COMMITTED, STATUS_READY_FOR_RENAMING]: + error_message = 'cannot rename entry, invalid status: ' + self.entry_status + if log_enabled(ERROR): + log(ERROR, '%s for <%s>', error_message, self) + raise LDAPCursorError(error_message) + self._state._to = new_name + self._state.set_status(STATUS_READY_FOR_RENAMING) + + @property + def entry_changes(self): + return self._changes diff --git a/server/www/packages/packages-linux/x64/ldap3/abstract/objectDef.py b/server/www/packages/packages-linux/x64/ldap3/abstract/objectDef.py index 5af64d5..7fdc481 100644 --- a/server/www/packages/packages-linux/x64/ldap3/abstract/objectDef.py +++ b/server/www/packages/packages-linux/x64/ldap3/abstract/objectDef.py @@ -5,7 +5,7 @@ # # Author: Giovanni Cannata # -# Copyright 2014 - 2018 Giovanni Cannata +# Copyright 2014 - 2019 Giovanni Cannata # # This file is part of ldap3. # diff --git a/server/www/packages/packages-linux/x64/ldap3/core/connection.py b/server/www/packages/packages-linux/x64/ldap3/core/connection.py index b8ed002..9d9211a 100644 --- a/server/www/packages/packages-linux/x64/ldap3/core/connection.py +++ b/server/www/packages/packages-linux/x64/ldap3/core/connection.py @@ -5,7 +5,7 @@ # # Author: Giovanni Cannata # -# Copyright 2014 - 2018 Giovanni Cannata +# Copyright 2014 - 2019 Giovanni Cannata # # This file is part of ldap3. # @@ -30,8 +30,9 @@ import json from .. import ANONYMOUS, SIMPLE, SASL, MODIFY_ADD, MODIFY_DELETE, MODIFY_REPLACE, get_config_parameter, DEREF_ALWAYS, \ SUBTREE, ASYNC, SYNC, NO_ATTRIBUTES, ALL_ATTRIBUTES, ALL_OPERATIONAL_ATTRIBUTES, MODIFY_INCREMENT, LDIF, ASYNC_STREAM, \ - RESTARTABLE, ROUND_ROBIN, REUSABLE, AUTO_BIND_DEFAULT, AUTO_BIND_NONE, AUTO_BIND_TLS_BEFORE_BIND, AUTO_BIND_TLS_AFTER_BIND, AUTO_BIND_NO_TLS, \ - STRING_TYPES, SEQUENCE_TYPES, MOCK_SYNC, MOCK_ASYNC, NTLM, EXTERNAL, DIGEST_MD5, GSSAPI, PLAIN + RESTARTABLE, ROUND_ROBIN, REUSABLE, AUTO_BIND_DEFAULT, AUTO_BIND_NONE, AUTO_BIND_TLS_BEFORE_BIND,\ + AUTO_BIND_TLS_AFTER_BIND, AUTO_BIND_NO_TLS, STRING_TYPES, SEQUENCE_TYPES, MOCK_SYNC, MOCK_ASYNC, NTLM, EXTERNAL,\ + DIGEST_MD5, GSSAPI, PLAIN from .results import RESULT_SUCCESS, RESULT_COMPARE_TRUE, RESULT_COMPARE_FALSE from ..extend import ExtendedOperationsRoot @@ -163,6 +164,8 @@ class Connection(object): :type pool_size: int :param pool_lifetime: pool lifetime for pooled strategies :type pool_lifetime: int + :param cred_store: credential store for gssapi + :type cred_store: dict :param use_referral_cache: keep referral connections open and reuse them :type use_referral_cache: bool :param auto_escape: automatic escaping of filter values @@ -190,6 +193,7 @@ class Connection(object): pool_name=None, pool_size=None, pool_lifetime=None, + cred_store=None, fast_decoder=True, receive_timeout=None, return_empty_attributes=True, @@ -254,6 +258,7 @@ class Connection(object): self.lazy = lazy self.pool_name = pool_name if pool_name else conf_default_pool_name self.pool_size = pool_size + self.cred_store = cred_store self.pool_lifetime = pool_lifetime self.pool_keepalive = pool_keepalive self.starting_tls = False @@ -333,7 +338,7 @@ class Connection(object): if log_enabled(BASIC): log(BASIC, 'performing automatic bind for <%s>', self) if self.closed: - self.open(read_server_info=False) + self.open(read_server_info=False) if self.auto_bind == AUTO_BIND_NO_TLS: self.bind(read_server_info=True) elif self.auto_bind == AUTO_BIND_TLS_BEFORE_BIND: @@ -387,6 +392,7 @@ class Connection(object): r += '' if self.pool_size is None else ', pool_size={0.pool_size!r}'.format(self) r += '' if self.pool_lifetime is None else ', pool_lifetime={0.pool_lifetime!r}'.format(self) r += '' if self.pool_keepalive is None else ', pool_keepalive={0.pool_keepalive!r}'.format(self) + r += '' if self.cred_store is None else (', cred_store=' + repr(self.cred_store)) r += '' if self.fast_decoder is None else (', fast_decoder=' + ('True' if self.fast_decoder else 'False')) r += '' if self.auto_range is None else (', auto_range=' + ('True' if self.auto_range else 'False')) r += '' if self.receive_timeout is None else ', receive_timeout={0.receive_timeout!r}'.format(self) @@ -425,6 +431,7 @@ class Connection(object): r += '' if self.pool_size is None else ', pool_size={0.pool_size!r}'.format(self) r += '' if self.pool_lifetime is None else ', pool_lifetime={0.pool_lifetime!r}'.format(self) r += '' if self.pool_keepalive is None else ', pool_keepalive={0.pool_keepalive!r}'.format(self) + r += '' if self.cred_store is None else (', cred_store=' + repr(self.cred_store)) r += '' if self.fast_decoder is None else (', fast_decoder=' + 'True' if self.fast_decoder else 'False') r += '' if self.auto_range is None else (', auto_range=' + ('True' if self.auto_range else 'False')) r += '' if self.receive_timeout is None else ', receive_timeout={0.receive_timeout!r}'.format(self) @@ -1031,6 +1038,7 @@ class Connection(object): log(ERROR, '%s for <%s>', self.last_error, self) raise LDAPChangeError(self.last_error) + changelist = dict() for attribute_name in changes: if self.server and self.server.schema and self.check_names: if ';' in attribute_name: # remove tags for checking @@ -1048,7 +1056,7 @@ class Connection(object): log(ERROR, '%s for <%s>', self.last_error, self) raise LDAPChangeError(self.last_error) - changes[attribute_name] = [change] # insert change in a tuple + changelist[attribute_name] = [change] # insert change in a list else: for change_operation in change: if len(change_operation) != 2 or change_operation[0] not in [MODIFY_ADD, MODIFY_DELETE, MODIFY_REPLACE, MODIFY_INCREMENT, 0, 1, 2, 3]: @@ -1056,7 +1064,8 @@ class Connection(object): if log_enabled(ERROR): log(ERROR, '%s for <%s>', self.last_error, self) raise LDAPChangeError(self.last_error) - request = modify_operation(dn, changes, self.auto_encode, self.server.schema if self.server else None, validator=self.server.custom_validator if self.server else None, check_names=self.check_names) + changelist[attribute_name] = change + request = modify_operation(dn, changelist, self.auto_encode, self.server.schema if self.server else None, validator=self.server.custom_validator if self.server else None, check_names=self.check_names) if log_enabled(PROTOCOL): log(PROTOCOL, 'MODIFY request <%s> sent via <%s>', modify_request_to_dict(request), self) response = self.post_send_single_response(self.send('modifyRequest', request, controls)) @@ -1107,11 +1116,11 @@ class Connection(object): log(ERROR, '%s for <%s>', self.last_error, self) raise LDAPConnectionIsReadOnlyError(self.last_error) - if new_superior and not dn.startswith(relative_dn): # as per RFC4511 (4.9) - self.last_error = 'DN cannot change while performing moving' - if log_enabled(ERROR): - log(ERROR, '%s for <%s>', self.last_error, self) - raise LDAPChangeError(self.last_error) + # if new_superior and not dn.startswith(relative_dn): # as per RFC4511 (4.9) + # self.last_error = 'DN cannot change while performing moving' + # if log_enabled(ERROR): + # log(ERROR, '%s for <%s>', self.last_error, self) + # raise LDAPChangeError(self.last_error) request = modify_dn_operation(dn, relative_dn, delete_old_dn, new_superior) if log_enabled(PROTOCOL): @@ -1220,6 +1229,8 @@ class Connection(object): log(BASIC, 'deferring START TLS for <%s>', self) else: self._deferred_start_tls = False + if self.closed: + self.open() if self.server.tls.start_tls(self) and self.strategy.sync: # for asynchronous connections _start_tls is run by the strategy if read_server_info: self.refresh_server_info() # refresh server info as per RFC4515 (3.1.5) @@ -1269,54 +1280,58 @@ class Connection(object): result = None if not self.sasl_in_progress: self.sasl_in_progress = True # ntlm is same of sasl authentication - # additional import for NTLM - from ..utils.ntlm import NtlmClient - domain_name, user_name = self.user.split('\\', 1) - ntlm_client = NtlmClient(user_name=user_name, domain=domain_name, password=self.password) + try: + # additional import for NTLM + from ..utils.ntlm import NtlmClient + domain_name, user_name = self.user.split('\\', 1) + ntlm_client = NtlmClient(user_name=user_name, domain=domain_name, password=self.password) - # as per https://msdn.microsoft.com/en-us/library/cc223501.aspx - # send a sicilyPackageDiscovery request (in the bindRequest) - request = bind_operation(self.version, 'SICILY_PACKAGE_DISCOVERY', ntlm_client) - if log_enabled(PROTOCOL): - log(PROTOCOL, 'NTLM SICILY PACKAGE DISCOVERY request sent via <%s>', self) - response = self.post_send_single_response(self.send('bindRequest', request, controls)) - if not self.strategy.sync: - _, result = self.get_response(response) - else: - result = response[0] - if 'server_creds' in result: - sicily_packages = result['server_creds'].decode('ascii').split(';') - if 'NTLM' in sicily_packages: # NTLM available on server - request = bind_operation(self.version, 'SICILY_NEGOTIATE_NTLM', ntlm_client) - if log_enabled(PROTOCOL): - log(PROTOCOL, 'NTLM SICILY NEGOTIATE request sent via <%s>', self) - response = self.post_send_single_response(self.send('bindRequest', request, controls)) - if not self.strategy.sync: - _, result = self.get_response(response) - else: + # as per https://msdn.microsoft.com/en-us/library/cc223501.aspx + # send a sicilyPackageDiscovery request (in the bindRequest) + request = bind_operation(self.version, 'SICILY_PACKAGE_DISCOVERY', ntlm_client) + if log_enabled(PROTOCOL): + log(PROTOCOL, 'NTLM SICILY PACKAGE DISCOVERY request sent via <%s>', self) + response = self.post_send_single_response(self.send('bindRequest', request, controls)) + if not self.strategy.sync: + _, result = self.get_response(response) + else: + result = response[0] + if 'server_creds' in result: + sicily_packages = result['server_creds'].decode('ascii').split(';') + if 'NTLM' in sicily_packages: # NTLM available on server + request = bind_operation(self.version, 'SICILY_NEGOTIATE_NTLM', ntlm_client) if log_enabled(PROTOCOL): - log(PROTOCOL, 'NTLM SICILY NEGOTIATE response <%s> received via <%s>', response[0], self) - result = response[0] - - if result['result'] == RESULT_SUCCESS: - request = bind_operation(self.version, 'SICILY_RESPONSE_NTLM', ntlm_client, result['server_creds']) - if log_enabled(PROTOCOL): - log(PROTOCOL, 'NTLM SICILY RESPONSE NTLM request sent via <%s>', self) + log(PROTOCOL, 'NTLM SICILY NEGOTIATE request sent via <%s>', self) response = self.post_send_single_response(self.send('bindRequest', request, controls)) if not self.strategy.sync: _, result = self.get_response(response) else: if log_enabled(PROTOCOL): - log(PROTOCOL, 'NTLM BIND response <%s> received via <%s>', response[0], self) + log(PROTOCOL, 'NTLM SICILY NEGOTIATE response <%s> received via <%s>', response[0], + self) result = response[0] - else: - result = None - self.sasl_in_progress = False - if log_enabled(BASIC): - log(BASIC, 'done SASL NTLM operation, result <%s>', result) + if result['result'] == RESULT_SUCCESS: + request = bind_operation(self.version, 'SICILY_RESPONSE_NTLM', ntlm_client, + result['server_creds']) + if log_enabled(PROTOCOL): + log(PROTOCOL, 'NTLM SICILY RESPONSE NTLM request sent via <%s>', self) + response = self.post_send_single_response(self.send('bindRequest', request, controls)) + if not self.strategy.sync: + _, result = self.get_response(response) + else: + if log_enabled(PROTOCOL): + log(PROTOCOL, 'NTLM BIND response <%s> received via <%s>', response[0], self) + result = response[0] + else: + result = None + finally: + self.sasl_in_progress = False - return result + if log_enabled(BASIC): + log(BASIC, 'done SASL NTLM operation, result <%s>', result) + + return result def refresh_server_info(self): # if self.strategy.no_real_dsa: # do not refresh for mock strategies diff --git a/server/www/packages/packages-linux/x64/ldap3/core/exceptions.py b/server/www/packages/packages-linux/x64/ldap3/core/exceptions.py index cfefb6d..b5d22b9 100644 --- a/server/www/packages/packages-linux/x64/ldap3/core/exceptions.py +++ b/server/www/packages/packages-linux/x64/ldap3/core/exceptions.py @@ -5,7 +5,7 @@ # # Author: Giovanni Cannata # -# Copyright 2014 - 2018 Giovanni Cannata +# Copyright 2014 - 2019 Giovanni Cannata # # This file is part of ldap3. # @@ -376,9 +376,11 @@ class LDAPAttributeError(LDAPExceptionError, ValueError, TypeError): class LDAPCursorError(LDAPExceptionError): pass + class LDAPObjectDereferenceError(LDAPExceptionError): pass + # security exceptions class LDAPSSLNotSupportedError(LDAPExceptionError, ImportError): pass @@ -505,6 +507,10 @@ class LDAPTransactionError(LDAPExceptionError): pass +class LDAPInfoError(LDAPExceptionError): + pass + + # communication exceptions class LDAPCommunicationError(LDAPExceptionError): pass diff --git a/server/www/packages/packages-linux/x64/ldap3/core/pooling.py b/server/www/packages/packages-linux/x64/ldap3/core/pooling.py index 66a0bbd..890023c 100644 --- a/server/www/packages/packages-linux/x64/ldap3/core/pooling.py +++ b/server/www/packages/packages-linux/x64/ldap3/core/pooling.py @@ -1,306 +1,329 @@ -""" -""" - -# Created on 2014.03.14 -# -# Author: Giovanni Cannata -# -# Copyright 2014 - 2018 Giovanni Cannata -# -# This file is part of ldap3. -# -# ldap3 is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# ldap3 is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with ldap3 in the COPYING and COPYING.LESSER files. -# If not, see . - -from datetime import datetime, MINYEAR -from os import linesep -from random import randint -from time import sleep - -from .. import FIRST, ROUND_ROBIN, RANDOM, SEQUENCE_TYPES, STRING_TYPES, get_config_parameter -from .exceptions import LDAPUnknownStrategyError, LDAPServerPoolError, LDAPServerPoolExhaustedError -from .server import Server -from ..utils.log import log, log_enabled, ERROR, BASIC, NETWORK - -POOLING_STRATEGIES = [FIRST, ROUND_ROBIN, RANDOM] - - -class ServerPoolState(object): - def __init__(self, server_pool): - self.servers = [] # each element is a list: [server, last_checked_time, available] - self.strategy = server_pool.strategy - self.server_pool = server_pool - self.last_used_server = 0 - self.refresh() - self.initialize_time = datetime.now() - - if log_enabled(BASIC): - log(BASIC, 'instantiated ServerPoolState: <%r>', self) - - def __str__(self): - s = 'servers: ' + linesep - if self.servers: - for server in self.servers: - s += str(server[0]) + linesep - else: - s += 'None' + linesep - s += 'Pool strategy: ' + str(self.strategy) + linesep - s += ' - Last used server: ' + ('None' if self.last_used_server == -1 else str(self.servers[self.last_used_server][0])) - - return s - - def refresh(self): - self.servers = [] - for server in self.server_pool.servers: - self.servers.append([server, datetime(MINYEAR, 1, 1), True]) # server, smallest date ever, supposed available - self.last_used_server = randint(0, len(self.servers) - 1) - - def get_current_server(self): - return self.servers[self.last_used_server][0] - - def get_server(self): - if self.servers: - if self.server_pool.strategy == FIRST: - if self.server_pool.active: - # returns the first active server - self.last_used_server = self.find_active_server(starting=0) - else: - # returns always the first server - no pooling - self.last_used_server = 0 - elif self.server_pool.strategy == ROUND_ROBIN: - if self.server_pool.active: - # returns the next active server in a circular range - self.last_used_server = self.find_active_server(self.last_used_server + 1) - else: - # returns the next server in a circular range - self.last_used_server = self.last_used_server + 1 if (self.last_used_server + 1) < len(self.servers) else 0 - elif self.server_pool.strategy == RANDOM: - if self.server_pool.active: - self.last_used_server = self.find_active_random_server() - else: - # returns a random server in the pool - self.last_used_server = randint(0, len(self.servers) - 1) - else: - if log_enabled(ERROR): - log(ERROR, 'unknown server pooling strategy <%s>', self.server_pool.strategy) - raise LDAPUnknownStrategyError('unknown server pooling strategy') - if log_enabled(BASIC): - log(BASIC, 'server returned from Server Pool: <%s>', self.last_used_server) - return self.servers[self.last_used_server][0] - else: - if log_enabled(ERROR): - log(ERROR, 'no servers in Server Pool <%s>', self) - raise LDAPServerPoolError('no servers in server pool') - - def find_active_random_server(self): - counter = self.server_pool.active # can be True for "forever" or the number of cycles to try - while counter: - if log_enabled(NETWORK): - log(NETWORK, 'entering loop for finding active server in pool <%s>', self) - temp_list = self.servers[:] # copy - while temp_list: - # pops a random server from a temp list and checks its - # availability, if not available tries another one - server = temp_list.pop(randint(0, len(temp_list) - 1)) - if not server[2]: # server is offline - if (isinstance(self.server_pool.exhaust, bool) and self.server_pool.exhaust) or (datetime.now() - server[1]).seconds < self.server_pool.exhaust: # keeps server offline - if log_enabled(NETWORK): - log(NETWORK, 'server <%s> excluded from checking because it is offline', server[0]) - continue - if log_enabled(NETWORK): - log(NETWORK, 'server <%s> reinserted in pool', server[0]) - server[1] = datetime.now() - if log_enabled(NETWORK): - log(NETWORK, 'checking server <%s> for availability', server[0]) - if server[0].check_availability(): - # returns a random active server in the pool - server[2] = True - return self.servers.index(server) - else: - server[2] = False - if not isinstance(self.server_pool.active, bool): - counter -= 1 - if log_enabled(ERROR): - log(ERROR, 'no random active server available in Server Pool <%s> after maximum number of tries', self) - raise LDAPServerPoolExhaustedError('no random active server available in server pool after maximum number of tries') - - def find_active_server(self, starting): - conf_pool_timeout = get_config_parameter('POOLING_LOOP_TIMEOUT') - counter = self.server_pool.active # can be True for "forever" or the number of cycles to try - if starting >= len(self.servers): - starting = 0 - - while counter: - if log_enabled(NETWORK): - log(NETWORK, 'entering loop number <%s> for finding active server in pool <%s>', counter, self) - index = -1 - pool_size = len(self.servers) - while index < pool_size - 1: - index += 1 - offset = index + starting if index + starting < pool_size else index + starting - pool_size - if not self.servers[offset][2]: # server is offline - if (isinstance(self.server_pool.exhaust, bool) and self.server_pool.exhaust) or (datetime.now() - self.servers[offset][1]).seconds < self.server_pool.exhaust: # keeps server offline - if log_enabled(NETWORK): - if isinstance(self.server_pool.exhaust, bool): - log(NETWORK, 'server <%s> excluded from checking because is offline', self.servers[offset][0]) - else: - log(NETWORK, 'server <%s> excluded from checking because is offline for %d seconds', self.servers[offset][0], (self.server_pool.exhaust - (datetime.now() - self.servers[offset][1]).seconds)) - continue - if log_enabled(NETWORK): - log(NETWORK, 'server <%s> reinserted in pool', self.servers[offset][0]) - self.servers[offset][1] = datetime.now() - if log_enabled(NETWORK): - log(NETWORK, 'checking server <%s> for availability', self.servers[offset][0]) - if self.servers[offset][0].check_availability(): - self.servers[offset][2] = True - return offset - else: - self.servers[offset][2] = False # sets server offline - - if not isinstance(self.server_pool.active, bool): - counter -= 1 - if log_enabled(NETWORK): - log(NETWORK, 'waiting for %d seconds before retrying pool servers cycle', conf_pool_timeout) - sleep(conf_pool_timeout) - - if log_enabled(ERROR): - log(ERROR, 'no active server available in Server Pool <%s> after maximum number of tries', self) - raise LDAPServerPoolExhaustedError('no active server available in server pool after maximum number of tries') - - def __len__(self): - return len(self.servers) - - -class ServerPool(object): - def __init__(self, - servers=None, - pool_strategy=ROUND_ROBIN, - active=True, - exhaust=False): - - if pool_strategy not in POOLING_STRATEGIES: - if log_enabled(ERROR): - log(ERROR, 'unknown pooling strategy <%s>', pool_strategy) - raise LDAPUnknownStrategyError('unknown pooling strategy') - if exhaust and not active: - if log_enabled(ERROR): - log(ERROR, 'cannot instantiate pool with exhaust and not active') - raise LDAPServerPoolError('pools can be exhausted only when checking for active servers') - self.servers = [] - self.pool_states = dict() - self.active = active - self.exhaust = exhaust - if isinstance(servers, SEQUENCE_TYPES + (Server, )): - self.add(servers) - elif isinstance(servers, STRING_TYPES): - self.add(Server(servers)) - self.strategy = pool_strategy - - if log_enabled(BASIC): - log(BASIC, 'instantiated ServerPool: <%r>', self) - - def __str__(self): - s = 'servers: ' + linesep - if self.servers: - for server in self.servers: - s += str(server) + linesep - else: - s += 'None' + linesep - s += 'Pool strategy: ' + str(self.strategy) - s += ' - ' + 'active: ' + (str(self.active) if self.active else 'False') - s += ' - ' + 'exhaust pool: ' + (str(self.exhaust) if self.exhaust else 'False') - return s - - def __repr__(self): - r = 'ServerPool(servers=' - if self.servers: - r += '[' - for server in self.servers: - r += server.__repr__() + ', ' - r = r[:-2] + ']' - else: - r += 'None' - r += ', pool_strategy={0.strategy!r}'.format(self) - r += ', active={0.active!r}'.format(self) - r += ', exhaust={0.exhaust!r}'.format(self) - r += ')' - - return r - - def __len__(self): - return len(self.servers) - - def __getitem__(self, item): - return self.servers[item] - - def __iter__(self): - return self.servers.__iter__() - - def add(self, servers): - if isinstance(servers, Server): - if servers not in self.servers: - self.servers.append(servers) - elif isinstance(servers, STRING_TYPES): - self.servers.append(Server(servers)) - elif isinstance(servers, SEQUENCE_TYPES): - for server in servers: - if isinstance(server, Server): - self.servers.append(server) - elif isinstance(server, STRING_TYPES): - self.servers.append(Server(server)) - else: - if log_enabled(ERROR): - log(ERROR, 'element must be a server in Server Pool <%s>', self) - raise LDAPServerPoolError('server in ServerPool must be a Server') - else: - if log_enabled(ERROR): - log(ERROR, 'server must be a Server of a list of Servers when adding to Server Pool <%s>', self) - raise LDAPServerPoolError('server must be a Server or a list of Server') - - for connection in self.pool_states: - # notifies connections using this pool to refresh - self.pool_states[connection].refresh() - - def remove(self, server): - if server in self.servers: - self.servers.remove(server) - else: - if log_enabled(ERROR): - log(ERROR, 'server %s to be removed not in Server Pool <%s>', server, self) - raise LDAPServerPoolError('server not in server pool') - - for connection in self.pool_states: - # notifies connections using this pool to refresh - self.pool_states[connection].refresh() - - def initialize(self, connection): - pool_state = ServerPoolState(self) - # registers pool_state in ServerPool object - self.pool_states[connection] = pool_state - - def get_server(self, connection): - if connection in self.pool_states: - return self.pool_states[connection].get_server() - else: - if log_enabled(ERROR): - log(ERROR, 'connection <%s> not in Server Pool State <%s>', connection, self) - raise LDAPServerPoolError('connection not in ServerPoolState') - - def get_current_server(self, connection): - if connection in self.pool_states: - return self.pool_states[connection].get_current_server() - else: - if log_enabled(ERROR): - log(ERROR, 'connection <%s> not in Server Pool State <%s>', connection, self) - raise LDAPServerPoolError('connection not in ServerPoolState') +""" +""" + +# Created on 2014.03.14 +# +# Author: Giovanni Cannata +# +# Copyright 2014 - 2019 Giovanni Cannata +# +# This file is part of ldap3. +# +# ldap3 is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ldap3 is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with ldap3 in the COPYING and COPYING.LESSER files. +# If not, see . + +from datetime import datetime, MINYEAR +from os import linesep +from random import randint +from time import sleep + +from .. import FIRST, ROUND_ROBIN, RANDOM, SEQUENCE_TYPES, STRING_TYPES, get_config_parameter +from .exceptions import LDAPUnknownStrategyError, LDAPServerPoolError, LDAPServerPoolExhaustedError +from .server import Server +from ..utils.log import log, log_enabled, ERROR, BASIC, NETWORK + +POOLING_STRATEGIES = [FIRST, ROUND_ROBIN, RANDOM] + + +class ServerState(object): + def __init__(self, server, last_checked_time, available): + self.server = server + self.last_checked_time = last_checked_time + self.available = available + + +class ServerPoolState(object): + def __init__(self, server_pool): + self.server_states = [] # each element is a ServerState + self.strategy = server_pool.strategy + self.server_pool = server_pool + self.last_used_server = 0 + self.refresh() + self.initialize_time = datetime.now() + + if log_enabled(BASIC): + log(BASIC, 'instantiated ServerPoolState: <%r>', self) + + def __str__(self): + s = 'servers: ' + linesep + if self.server_states: + for state in self.server_states: + s += str(state.server) + linesep + else: + s += 'None' + linesep + s += 'Pool strategy: ' + str(self.strategy) + linesep + s += ' - Last used server: ' + ('None' if self.last_used_server == -1 else str(self.server_states[self.last_used_server].server)) + + return s + + def refresh(self): + self.server_states = [] + for server in self.server_pool.servers: + self.server_states.append(ServerState(server, datetime(MINYEAR, 1, 1), True)) # server, smallest date ever, supposed available + self.last_used_server = randint(0, len(self.server_states) - 1) + + def get_current_server(self): + return self.server_states[self.last_used_server].server + + def get_server(self): + if self.server_states: + if self.server_pool.strategy == FIRST: + if self.server_pool.active: + # returns the first active server + self.last_used_server = self.find_active_server(starting=0) + else: + # returns always the first server - no pooling + self.last_used_server = 0 + elif self.server_pool.strategy == ROUND_ROBIN: + if self.server_pool.active: + # returns the next active server in a circular range + self.last_used_server = self.find_active_server(self.last_used_server + 1) + else: + # returns the next server in a circular range + self.last_used_server = self.last_used_server + 1 if (self.last_used_server + 1) < len(self.server_states) else 0 + elif self.server_pool.strategy == RANDOM: + if self.server_pool.active: + self.last_used_server = self.find_active_random_server() + else: + # returns a random server in the pool + self.last_used_server = randint(0, len(self.server_states) - 1) + else: + if log_enabled(ERROR): + log(ERROR, 'unknown server pooling strategy <%s>', self.server_pool.strategy) + raise LDAPUnknownStrategyError('unknown server pooling strategy') + if log_enabled(BASIC): + log(BASIC, 'server returned from Server Pool: <%s>', self.last_used_server) + return self.server_states[self.last_used_server].server + else: + if log_enabled(ERROR): + log(ERROR, 'no servers in Server Pool <%s>', self) + raise LDAPServerPoolError('no servers in server pool') + + def find_active_random_server(self): + counter = self.server_pool.active # can be True for "forever" or the number of cycles to try + while counter: + if log_enabled(NETWORK): + log(NETWORK, 'entering loop for finding active server in pool <%s>', self) + temp_list = self.server_states[:] # copy + while temp_list: + # pops a random server from a temp list and checks its + # availability, if not available tries another one + server_state = temp_list.pop(randint(0, len(temp_list) - 1)) + if not server_state.available: # server is offline + if (isinstance(self.server_pool.exhaust, bool) and self.server_pool.exhaust) or (datetime.now() - server_state.last_checked_time).seconds < self.server_pool.exhaust: # keeps server offline + if log_enabled(NETWORK): + log(NETWORK, 'server <%s> excluded from checking because it is offline', server_state.server) + continue + if log_enabled(NETWORK): + log(NETWORK, 'server <%s> reinserted in pool', server_state.server) + server_state.last_checked_time = datetime.now() + if log_enabled(NETWORK): + log(NETWORK, 'checking server <%s> for availability', server_state.server) + if server_state.server.check_availability(): + # returns a random active server in the pool + server_state.available = True + return self.server_states.index(server_state) + else: + server_state.available = False + if not isinstance(self.server_pool.active, bool): + counter -= 1 + if log_enabled(ERROR): + log(ERROR, 'no random active server available in Server Pool <%s> after maximum number of tries', self) + raise LDAPServerPoolExhaustedError('no random active server available in server pool after maximum number of tries') + + def find_active_server(self, starting): + conf_pool_timeout = get_config_parameter('POOLING_LOOP_TIMEOUT') + counter = self.server_pool.active # can be True for "forever" or the number of cycles to try + if starting >= len(self.server_states): + starting = 0 + + while counter: + if log_enabled(NETWORK): + log(NETWORK, 'entering loop number <%s> for finding active server in pool <%s>', counter, self) + index = -1 + pool_size = len(self.server_states) + while index < pool_size - 1: + index += 1 + offset = index + starting if index + starting < pool_size else index + starting - pool_size + server_state = self.server_states[offset] + if not server_state.available: # server is offline + if (isinstance(self.server_pool.exhaust, bool) and self.server_pool.exhaust) or (datetime.now() - server_state.last_checked_time).seconds < self.server_pool.exhaust: # keeps server offline + if log_enabled(NETWORK): + if isinstance(self.server_pool.exhaust, bool): + log(NETWORK, 'server <%s> excluded from checking because is offline', server_state.server) + else: + log(NETWORK, 'server <%s> excluded from checking because is offline for %d seconds', server_state.server, (self.server_pool.exhaust - (datetime.now() - server_state.last_checked_time).seconds)) + continue + if log_enabled(NETWORK): + log(NETWORK, 'server <%s> reinserted in pool', server_state.server) + server_state.last_checked_time = datetime.now() + if log_enabled(NETWORK): + log(NETWORK, 'checking server <%s> for availability', server_state.server) + if server_state.server.check_availability(): + server_state.available = True + return offset + else: + server_state.available = False # sets server offline + + if not isinstance(self.server_pool.active, bool): + counter -= 1 + if log_enabled(NETWORK): + log(NETWORK, 'waiting for %d seconds before retrying pool servers cycle', conf_pool_timeout) + sleep(conf_pool_timeout) + + if log_enabled(ERROR): + log(ERROR, 'no active server available in Server Pool <%s> after maximum number of tries', self) + raise LDAPServerPoolExhaustedError('no active server available in server pool after maximum number of tries') + + def __len__(self): + return len(self.server_states) + + +class ServerPool(object): + def __init__(self, + servers=None, + pool_strategy=ROUND_ROBIN, + active=True, + exhaust=False, + single_state=True): + + if pool_strategy not in POOLING_STRATEGIES: + if log_enabled(ERROR): + log(ERROR, 'unknown pooling strategy <%s>', pool_strategy) + raise LDAPUnknownStrategyError('unknown pooling strategy') + if exhaust and not active: + if log_enabled(ERROR): + log(ERROR, 'cannot instantiate pool with exhaust and not active') + raise LDAPServerPoolError('pools can be exhausted only when checking for active servers') + self.servers = [] + self.pool_states = dict() + self.active = active + self.exhaust = exhaust + self.single = single_state + self._pool_state = None # used for storing the global state of the pool + if isinstance(servers, SEQUENCE_TYPES + (Server, )): + self.add(servers) + elif isinstance(servers, STRING_TYPES): + self.add(Server(servers)) + self.strategy = pool_strategy + + if log_enabled(BASIC): + log(BASIC, 'instantiated ServerPool: <%r>', self) + + def __str__(self): + s = 'servers: ' + linesep + if self.servers: + for server in self.servers: + s += str(server) + linesep + else: + s += 'None' + linesep + s += 'Pool strategy: ' + str(self.strategy) + s += ' - ' + 'active: ' + (str(self.active) if self.active else 'False') + s += ' - ' + 'exhaust pool: ' + (str(self.exhaust) if self.exhaust else 'False') + return s + + def __repr__(self): + r = 'ServerPool(servers=' + if self.servers: + r += '[' + for server in self.servers: + r += server.__repr__() + ', ' + r = r[:-2] + ']' + else: + r += 'None' + r += ', pool_strategy={0.strategy!r}'.format(self) + r += ', active={0.active!r}'.format(self) + r += ', exhaust={0.exhaust!r}'.format(self) + r += ')' + + return r + + def __len__(self): + return len(self.servers) + + def __getitem__(self, item): + return self.servers[item] + + def __iter__(self): + return self.servers.__iter__() + + def add(self, servers): + if isinstance(servers, Server): + if servers not in self.servers: + self.servers.append(servers) + elif isinstance(servers, STRING_TYPES): + self.servers.append(Server(servers)) + elif isinstance(servers, SEQUENCE_TYPES): + for server in servers: + if isinstance(server, Server): + self.servers.append(server) + elif isinstance(server, STRING_TYPES): + self.servers.append(Server(server)) + else: + if log_enabled(ERROR): + log(ERROR, 'element must be a server in Server Pool <%s>', self) + raise LDAPServerPoolError('server in ServerPool must be a Server') + else: + if log_enabled(ERROR): + log(ERROR, 'server must be a Server of a list of Servers when adding to Server Pool <%s>', self) + raise LDAPServerPoolError('server must be a Server or a list of Server') + + if self.single: + if self._pool_state: + self._pool_state.refresh() + else: + for connection in self.pool_states: + # notifies connections using this pool to refresh + self.pool_states[connection].refresh() + + def remove(self, server): + if server in self.servers: + self.servers.remove(server) + else: + if log_enabled(ERROR): + log(ERROR, 'server %s to be removed not in Server Pool <%s>', server, self) + raise LDAPServerPoolError('server not in server pool') + + if self.single: + if self._pool_state: + self._pool_state.refresh() + else: + for connection in self.pool_states: + # notifies connections using this pool to refresh + self.pool_states[connection].refresh() + + def initialize(self, connection): + # registers pool_state in ServerPool object + if self.single: + if not self._pool_state: + self._pool_state = ServerPoolState(self) + self.pool_states[connection] = self._pool_state + else: + self.pool_states[connection] = ServerPoolState(self) + + def get_server(self, connection): + if connection in self.pool_states: + return self.pool_states[connection].get_server() + else: + if log_enabled(ERROR): + log(ERROR, 'connection <%s> not in Server Pool State <%s>', connection, self) + raise LDAPServerPoolError('connection not in ServerPoolState') + + def get_current_server(self, connection): + if connection in self.pool_states: + return self.pool_states[connection].get_current_server() + else: + if log_enabled(ERROR): + log(ERROR, 'connection <%s> not in Server Pool State <%s>', connection, self) + raise LDAPServerPoolError('connection not in ServerPoolState') diff --git a/server/www/packages/packages-linux/x64/ldap3/core/results.py b/server/www/packages/packages-linux/x64/ldap3/core/results.py index 6f10643..4c08172 100644 --- a/server/www/packages/packages-linux/x64/ldap3/core/results.py +++ b/server/www/packages/packages-linux/x64/ldap3/core/results.py @@ -5,7 +5,7 @@ # # Author: Giovanni Cannata # -# Copyright 2014 - 2018 Giovanni Cannata +# Copyright 2014 - 2019 Giovanni Cannata # # This file is part of ldap3. # @@ -134,4 +134,4 @@ RESULT_CODES = { } # do not raise exception for (in raise_exceptions connection mode) -DO_NOT_RAISE_EXCEPTIONS = [RESULT_SUCCESS, RESULT_COMPARE_FALSE, RESULT_COMPARE_TRUE, RESULT_REFERRAL, RESULT_SASL_BIND_IN_PROGRESS] +DO_NOT_RAISE_EXCEPTIONS = [RESULT_SUCCESS, RESULT_COMPARE_FALSE, RESULT_COMPARE_TRUE, RESULT_REFERRAL, RESULT_SASL_BIND_IN_PROGRESS, RESULT_SIZE_LIMIT_EXCEEDED, RESULT_TIME_LIMIT_EXCEEDED] diff --git a/server/www/packages/packages-linux/x64/ldap3/core/server.py b/server/www/packages/packages-linux/x64/ldap3/core/server.py index 36c782b..c266f1c 100644 --- a/server/www/packages/packages-linux/x64/ldap3/core/server.py +++ b/server/www/packages/packages-linux/x64/ldap3/core/server.py @@ -5,7 +5,7 @@ # # Author: Giovanni Cannata # -# Copyright 2014 - 2018 Giovanni Cannata +# Copyright 2014 - 2019 Giovanni Cannata # # This file is part of ldap3. # @@ -27,8 +27,8 @@ import socket from threading import Lock from datetime import datetime, MINYEAR -from .. import DSA, SCHEMA, ALL, BASE, get_config_parameter, OFFLINE_EDIR_8_8_8, OFFLINE_AD_2012_R2, OFFLINE_SLAPD_2_4, OFFLINE_DS389_1_3_3, SEQUENCE_TYPES, IP_SYSTEM_DEFAULT, IP_V4_ONLY, IP_V6_ONLY, IP_V4_PREFERRED, IP_V6_PREFERRED, STRING_TYPES -from .exceptions import LDAPInvalidServerError, LDAPDefinitionError, LDAPInvalidPortError, LDAPInvalidTlsSpecificationError, LDAPSocketOpenError +from .. import DSA, SCHEMA, ALL, BASE, get_config_parameter, OFFLINE_EDIR_8_8_8, OFFLINE_EDIR_9_1_4, OFFLINE_AD_2012_R2, OFFLINE_SLAPD_2_4, OFFLINE_DS389_1_3_3, SEQUENCE_TYPES, IP_SYSTEM_DEFAULT, IP_V4_ONLY, IP_V6_ONLY, IP_V4_PREFERRED, IP_V6_PREFERRED, STRING_TYPES +from .exceptions import LDAPInvalidServerError, LDAPDefinitionError, LDAPInvalidPortError, LDAPInvalidTlsSpecificationError, LDAPSocketOpenError, LDAPInfoError from ..protocol.formatters.standard import format_attribute_values from ..protocol.rfc4511 import LDAP_MAX_INT from ..protocol.rfc4512 import SchemaInfo, DsaInfo @@ -69,7 +69,6 @@ class Server(object): _message_counter = 0 _message_id_lock = Lock() # global lock for message_id shared by all Server objects - def __init__(self, host, port=None, @@ -452,6 +451,10 @@ class Server(object): from ..protocol.schemas.edir888 import edir_8_8_8_schema, edir_8_8_8_dsa_info self.attach_schema_info(SchemaInfo.from_json(edir_8_8_8_schema)) self.attach_dsa_info(DsaInfo.from_json(edir_8_8_8_dsa_info)) + elif self.get_info == OFFLINE_EDIR_9_1_4: + from ..protocol.schemas.edir914 import edir_9_1_4_schema, edir_9_1_4_dsa_info + self.attach_schema_info(SchemaInfo.from_json(edir_9_1_4_schema)) + self.attach_dsa_info(DsaInfo.from_json(edir_9_1_4_dsa_info)) elif self.get_info == OFFLINE_AD_2012_R2: from ..protocol.schemas.ad2012R2 import ad_2012_r2_schema, ad_2012_r2_dsa_info self.attach_schema_info(SchemaInfo.from_json(ad_2012_r2_schema)) @@ -570,3 +573,34 @@ class Server(object): for candidate in candidates: log(BASIC, 'obtained candidate address for <%s>: <%r> with mode %s', self, candidate[:-2], self.mode) return candidates + + def _check_info_property(self, kind, name): + if not self._dsa_info: + raise LDAPInfoError('server info not loaded') + + if kind == 'control': + properties = self.info.supported_controls + elif kind == 'extension': + properties = self.info.supported_extensions + elif kind == 'feature': + properties = self.info.supported_features + else: + raise LDAPInfoError('invalid info category') + + for prop in properties: + if name == prop[0] or (prop[2] and name.lower() == prop[2].lower()): # checks oid and description + return True + + return False + + def has_control(self, control): + return self._check_info_property('control', control) + + def has_extension(self, extension): + return self._check_info_property('extension', extension) + + def has_feature(self, feature): + return self._check_info_property('feature', feature) + + + diff --git a/server/www/packages/packages-linux/x64/ldap3/extend/__init__.py b/server/www/packages/packages-linux/x64/ldap3/extend/__init__.py index 24f426e..28e5f64 100644 --- a/server/www/packages/packages-linux/x64/ldap3/extend/__init__.py +++ b/server/www/packages/packages-linux/x64/ldap3/extend/__init__.py @@ -5,7 +5,7 @@ # # Author: Giovanni Cannata # -# Copyright 2014 - 2018 Giovanni Cannata +# Copyright 2014 - 2019 Giovanni Cannata # # This file is part of ldap3. # @@ -169,6 +169,36 @@ class StandardExtendedOperations(ExtendedOperationContainer): streaming, callback) + def funnel_search(self, + search_base='', + search_filter='', + search_scope=SUBTREE, + dereference_aliases=DEREF_NEVER, + attributes=ALL_ATTRIBUTES, + size_limit=0, + time_limit=0, + controls=None, + streaming=True, + callback=None, + queue_size=8 + ): + if callback: + streaming = False + return PersistentSearch(self._connection, + search_base, + search_filter, + search_scope, + dereference_aliases, + attributes, + size_limit, + time_limit, + controls, + None, + None, + None, + streaming, + callback) + class NovellExtendedOperations(ExtendedOperationContainer): def get_bind_dn(self, controls=None): diff --git a/server/www/packages/packages-linux/x64/ldap3/extend/microsoft/addMembersToGroups.py b/server/www/packages/packages-linux/x64/ldap3/extend/microsoft/addMembersToGroups.py index 28c409f..d7938fd 100644 --- a/server/www/packages/packages-linux/x64/ldap3/extend/microsoft/addMembersToGroups.py +++ b/server/www/packages/packages-linux/x64/ldap3/extend/microsoft/addMembersToGroups.py @@ -1,81 +1,93 @@ -""" -""" - -# Created on 2016.12.26 -# -# Author: Giovanni Cannata -# -# Copyright 2016 - 2018 Giovanni Cannata -# -# This file is part of ldap3. -# -# ldap3 is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# ldap3 is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with ldap3 in the COPYING and COPYING.LESSER files. -# If not, see . -from ...core.exceptions import LDAPInvalidDnError -from ... import SEQUENCE_TYPES, MODIFY_ADD, BASE, DEREF_NEVER - - -def ad_add_members_to_groups(connection, - members_dn, - groups_dn, - fix=True): - """ - :param connection: a bound Connection object - :param members_dn: the list of members to add to groups - :param groups_dn: the list of groups where members are to be added - :param fix: checks for group existence and already assigned members - :return: a boolean where True means that the operation was successful and False means an error has happened - Establishes users-groups relations following the Active Directory rules: users are added to the member attribute of groups. - Raises LDAPInvalidDnError if members or groups are not found in the DIT. - """ - - if not isinstance(members_dn, SEQUENCE_TYPES): - members_dn = [members_dn] - - if not isinstance(groups_dn, SEQUENCE_TYPES): - groups_dn = [groups_dn] - - error = False - for group in groups_dn: - if fix: # checks for existance of group and for already assigned members - result = connection.search(group, '(objectclass=*)', BASE, dereference_aliases=DEREF_NEVER, attributes=['member']) - - if not connection.strategy.sync: - response, result = connection.get_response(result) - else: - response, result = connection.response, connection.result - - if not result['description'] == 'success': - raise LDAPInvalidDnError(group + ' not found') - - existing_members = response[0]['attributes']['member'] if 'member' in response[0]['attributes'] else [] - existing_members = [element.lower() for element in existing_members] - else: - existing_members = [] - - changes = dict() - member_to_add = [element for element in members_dn if element.lower() not in existing_members] - if member_to_add: - changes['member'] = (MODIFY_ADD, member_to_add) - if changes: - result = connection.modify(group, changes) - if not connection.strategy.sync: - _, result = connection.get_response(result) - else: - result = connection.result - if result['description'] != 'success': - error = True - break - - return not error # returns True if no error is raised in the LDAP operations +""" +""" + +# Created on 2016.12.26 +# +# Author: Giovanni Cannata +# +# Copyright 2016 - 2018 Giovanni Cannata +# +# This file is part of ldap3. +# +# ldap3 is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ldap3 is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with ldap3 in the COPYING and COPYING.LESSER files. +# If not, see . + +from ... import SEQUENCE_TYPES, MODIFY_ADD, BASE, DEREF_NEVER +from ...core.exceptions import LDAPInvalidDnError, LDAPOperationsErrorResult +from ...utils.dn import safe_dn + + +def ad_add_members_to_groups(connection, + members_dn, + groups_dn, + fix=True, + raise_error=False): + """ + :param connection: a bound Connection object + :param members_dn: the list of members to add to groups + :param groups_dn: the list of groups where members are to be added + :param fix: checks for group existence and already assigned members + :param raise_error: If the operation fails it raises an error instead of returning False + :return: a boolean where True means that the operation was successful and False means an error has happened + Establishes users-groups relations following the Active Directory rules: users are added to the member attribute of groups. + Raises LDAPInvalidDnError if members or groups are not found in the DIT. + """ + + if not isinstance(members_dn, SEQUENCE_TYPES): + members_dn = [members_dn] + + if not isinstance(groups_dn, SEQUENCE_TYPES): + groups_dn = [groups_dn] + + if connection.check_names: # builds new lists with sanitized dn + members_dn = [safe_dn(member_dn) for member_dn in members_dn] + groups_dn = [safe_dn(group_dn) for group_dn in groups_dn] + + error = False + for group in groups_dn: + if fix: # checks for existance of group and for already assigned members + result = connection.search(group, '(objectclass=*)', BASE, dereference_aliases=DEREF_NEVER, + attributes=['member']) + + if not connection.strategy.sync: + response, result = connection.get_response(result) + else: + response, result = connection.response, connection.result + + if not result['description'] == 'success': + raise LDAPInvalidDnError(group + ' not found') + + existing_members = response[0]['attributes']['member'] if 'member' in response[0]['attributes'] else [] + existing_members = [element.lower() for element in existing_members] + else: + existing_members = [] + + changes = dict() + member_to_add = [element for element in members_dn if element.lower() not in existing_members] + if member_to_add: + changes['member'] = (MODIFY_ADD, member_to_add) + if changes: + result = connection.modify(group, changes) + if not connection.strategy.sync: + _, result = connection.get_response(result) + else: + result = connection.result + if result['description'] != 'success': + error = True + result_error_params = ['result', 'description', 'dn', 'message'] + if raise_error: + raise LDAPOperationsErrorResult([(k, v) for k, v in result.items() if k in result_error_params]) + break + + return not error # returns True if no error is raised in the LDAP operations diff --git a/server/www/packages/packages-linux/x64/ldap3/extend/microsoft/removeMembersFromGroups.py b/server/www/packages/packages-linux/x64/ldap3/extend/microsoft/removeMembersFromGroups.py index 1b7feb3..e415188 100644 --- a/server/www/packages/packages-linux/x64/ldap3/extend/microsoft/removeMembersFromGroups.py +++ b/server/www/packages/packages-linux/x64/ldap3/extend/microsoft/removeMembersFromGroups.py @@ -1,93 +1,92 @@ -""" -""" - -# Created on 2016.12.26 -# -# Author: Giovanni Cannata -# -# Copyright 2016 - 2018 Giovanni Cannata -# -# This file is part of ldap3. -# -# ldap3 is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# ldap3 is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with ldap3 in the COPYING and COPYING.LESSER files. -# If not, see . -from ...core.exceptions import LDAPInvalidDnError -from ... import SEQUENCE_TYPES, MODIFY_DELETE, BASE, DEREF_NEVER -from ...utils.dn import safe_dn - - -def ad_remove_members_from_groups(connection, - members_dn, - groups_dn, - fix): - """ - :param connection: a bound Connection object - :param members_dn: the list of members to remove from groups - :param groups_dn: the list of groups where members are to be removed - :param fix: checks for group existence and existing members - :return: a boolean where True means that the operation was successful and False means an error has happened - Removes users-groups relations following the Activwe Directory rules: users are removed from groups' member attribute - - """ - if not isinstance(members_dn, SEQUENCE_TYPES): - members_dn = [members_dn] - - if not isinstance(groups_dn, SEQUENCE_TYPES): - groups_dn = [groups_dn] - - if connection.check_names: # builds new lists with sanitized dn - safe_members_dn = [] - safe_groups_dn = [] - for member_dn in members_dn: - safe_members_dn.append(safe_dn(member_dn)) - for group_dn in groups_dn: - safe_groups_dn.append(safe_dn(group_dn)) - - members_dn = safe_members_dn - groups_dn = safe_groups_dn - - error = False - - for group in groups_dn: - if fix: # checks for existance of group and for already assigned members - result = connection.search(group, '(objectclass=*)', BASE, dereference_aliases=DEREF_NEVER, attributes=['member']) - - if not connection.strategy.sync: - response, result = connection.get_response(result) - else: - response, result = connection.response, connection.result - - if not result['description'] == 'success': - raise LDAPInvalidDnError(group + ' not found') - - existing_members = response[0]['attributes']['member'] if 'member' in response[0]['attributes'] else [] - else: - existing_members = members_dn - - existing_members = [element.lower() for element in existing_members] - changes = dict() - member_to_remove = [element for element in members_dn if element.lower() in existing_members] - if member_to_remove: - changes['member'] = (MODIFY_DELETE, member_to_remove) - if changes: - result = connection.modify(group, changes) - if not connection.strategy.sync: - _, result = connection.get_response(result) - else: - result = connection.result - if result['description'] != 'success': - error = True - break - - return not error +""" +""" + +# Created on 2016.12.26 +# +# Author: Giovanni Cannata +# +# Copyright 2016 - 2018 Giovanni Cannata +# +# This file is part of ldap3. +# +# ldap3 is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ldap3 is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with ldap3 in the COPYING and COPYING.LESSER files. +# If not, see . + +from ...core.exceptions import LDAPInvalidDnError, LDAPOperationsErrorResult +from ... import SEQUENCE_TYPES, MODIFY_DELETE, BASE, DEREF_NEVER +from ...utils.dn import safe_dn + + +def ad_remove_members_from_groups(connection, + members_dn, + groups_dn, + fix, + raise_error=False): + """ + :param connection: a bound Connection object + :param members_dn: the list of members to remove from groups + :param groups_dn: the list of groups where members are to be removed + :param fix: checks for group existence and existing members + :param raise_error: If the operation fails it raises an error instead of returning False + :return: a boolean where True means that the operation was successful and False means an error has happened + Removes users-groups relations following the Activwe Directory rules: users are removed from groups' member attribute + + """ + if not isinstance(members_dn, SEQUENCE_TYPES): + members_dn = [members_dn] + + if not isinstance(groups_dn, SEQUENCE_TYPES): + groups_dn = [groups_dn] + + if connection.check_names: # builds new lists with sanitized dn + members_dn = [safe_dn(member_dn) for member_dn in members_dn] + groups_dn = [safe_dn(group_dn) for group_dn in groups_dn] + + error = False + + for group in groups_dn: + if fix: # checks for existance of group and for already assigned members + result = connection.search(group, '(objectclass=*)', BASE, dereference_aliases=DEREF_NEVER, attributes=['member']) + + if not connection.strategy.sync: + response, result = connection.get_response(result) + else: + response, result = connection.response, connection.result + + if not result['description'] == 'success': + raise LDAPInvalidDnError(group + ' not found') + + existing_members = response[0]['attributes']['member'] if 'member' in response[0]['attributes'] else [] + else: + existing_members = members_dn + + existing_members = [element.lower() for element in existing_members] + changes = dict() + member_to_remove = [element for element in members_dn if element.lower() in existing_members] + if member_to_remove: + changes['member'] = (MODIFY_DELETE, member_to_remove) + if changes: + result = connection.modify(group, changes) + if not connection.strategy.sync: + _, result = connection.get_response(result) + else: + result = connection.result + if result['description'] != 'success': + error = True + result_error_params = ['result', 'description', 'dn', 'message'] + if raise_error: + raise LDAPOperationsErrorResult([(k, v) for k, v in result.items() if k in result_error_params]) + break + + return not error diff --git a/server/www/packages/packages-linux/x64/ldap3/extend/novell/getBindDn.py b/server/www/packages/packages-linux/x64/ldap3/extend/novell/getBindDn.py index 39fae2b..2bc4deb 100644 --- a/server/www/packages/packages-linux/x64/ldap3/extend/novell/getBindDn.py +++ b/server/www/packages/packages-linux/x64/ldap3/extend/novell/getBindDn.py @@ -5,7 +5,7 @@ # # Author: Giovanni Cannata # -# Copyright 2014 - 2018 Giovanni Cannata +# Copyright 2014 - 2019 Giovanni Cannata # # This file is part of ldap3. # diff --git a/server/www/packages/packages-linux/x64/ldap3/extend/novell/listReplicas.py b/server/www/packages/packages-linux/x64/ldap3/extend/novell/listReplicas.py index fdc6d08..804e8fe 100644 --- a/server/www/packages/packages-linux/x64/ldap3/extend/novell/listReplicas.py +++ b/server/www/packages/packages-linux/x64/ldap3/extend/novell/listReplicas.py @@ -5,7 +5,7 @@ # # Author: Giovanni Cannata # -# Copyright 2014 - 2018 Giovanni Cannata +# Copyright 2014 - 2019 Giovanni Cannata # # This file is part of ldap3. # @@ -45,6 +45,6 @@ class ListReplicas(ExtendedOperation): def populate_result(self): try: - self.result['replicas'] = str(self.decoded_response['replicaList']) if self.decoded_response['replicaList'] else None + self.result['replicas'] = [str(replica) for replica in self.decoded_response] if self.decoded_response else None except TypeError: self.result['replicas'] = None diff --git a/server/www/packages/packages-linux/x64/ldap3/extend/novell/nmasGetUniversalPassword.py b/server/www/packages/packages-linux/x64/ldap3/extend/novell/nmasGetUniversalPassword.py index b8b045b..a16b79f 100644 --- a/server/www/packages/packages-linux/x64/ldap3/extend/novell/nmasGetUniversalPassword.py +++ b/server/www/packages/packages-linux/x64/ldap3/extend/novell/nmasGetUniversalPassword.py @@ -5,7 +5,7 @@ # # Author: Giovanni Cannata # -# Copyright 2014 - 2018 Giovanni Cannata +# Copyright 2014 - 2019 Giovanni Cannata # # This file is part of ldap3. # diff --git a/server/www/packages/packages-linux/x64/ldap3/extend/novell/nmasSetUniversalPassword.py b/server/www/packages/packages-linux/x64/ldap3/extend/novell/nmasSetUniversalPassword.py index 65ea0d6..ee243ba 100644 --- a/server/www/packages/packages-linux/x64/ldap3/extend/novell/nmasSetUniversalPassword.py +++ b/server/www/packages/packages-linux/x64/ldap3/extend/novell/nmasSetUniversalPassword.py @@ -5,7 +5,7 @@ # # Author: Giovanni Cannata # -# Copyright 2014 - 2018 Giovanni Cannata +# Copyright 2014 - 2019 Giovanni Cannata # # This file is part of ldap3. # diff --git a/server/www/packages/packages-linux/x64/ldap3/extend/novell/partition_entry_count.py b/server/www/packages/packages-linux/x64/ldap3/extend/novell/partition_entry_count.py index 8218aea..185fa4d 100644 --- a/server/www/packages/packages-linux/x64/ldap3/extend/novell/partition_entry_count.py +++ b/server/www/packages/packages-linux/x64/ldap3/extend/novell/partition_entry_count.py @@ -5,7 +5,7 @@ # # Author: Giovanni Cannata # -# Copyright 2014 - 2018 Giovanni Cannata +# Copyright 2014 - 2019 Giovanni Cannata # # This file is part of ldap3. # diff --git a/server/www/packages/packages-linux/x64/ldap3/extend/novell/replicaInfo.py b/server/www/packages/packages-linux/x64/ldap3/extend/novell/replicaInfo.py index 45bd0e9..e331686 100644 --- a/server/www/packages/packages-linux/x64/ldap3/extend/novell/replicaInfo.py +++ b/server/www/packages/packages-linux/x64/ldap3/extend/novell/replicaInfo.py @@ -5,7 +5,7 @@ # # Author: Giovanni Cannata # -# Copyright 2014 - 2018 Giovanni Cannata +# Copyright 2014 - 2019 Giovanni Cannata # # This file is part of ldap3. # diff --git a/server/www/packages/packages-linux/x64/ldap3/extend/operation.py b/server/www/packages/packages-linux/x64/ldap3/extend/operation.py index 9906885..2998af3 100644 --- a/server/www/packages/packages-linux/x64/ldap3/extend/operation.py +++ b/server/www/packages/packages-linux/x64/ldap3/extend/operation.py @@ -5,7 +5,7 @@ # # Author: Giovanni Cannata # -# Copyright 2014 - 2018 Giovanni Cannata +# Copyright 2014 - 2019 Giovanni Cannata # # This file is part of ldap3. # diff --git a/server/www/packages/packages-linux/x64/ldap3/extend/standard/PagedSearch.py b/server/www/packages/packages-linux/x64/ldap3/extend/standard/PagedSearch.py index 1b5df49..30838df 100644 --- a/server/www/packages/packages-linux/x64/ldap3/extend/standard/PagedSearch.py +++ b/server/www/packages/packages-linux/x64/ldap3/extend/standard/PagedSearch.py @@ -5,7 +5,7 @@ # # Author: Giovanni Cannata # -# Copyright 2014 - 2018 Giovanni Cannata +# Copyright 2014 - 2019 Giovanni Cannata # # This file is part of ldap3. # @@ -47,7 +47,11 @@ def paged_search_generator(connection, search_base = safe_dn(search_base) responses = [] - cookie = True # performs search at least one time + original_connection = None + original_auto_referrals = connection.auto_referrals + connection.auto_referrals = False # disable auto referrals because it cannot handle paged searches + cookie = True # performs search operation at least one time + cachekey = None # for referrals cache while cookie: result = connection.search(search_base, search_filter, @@ -69,13 +73,19 @@ def paged_search_generator(connection, response = connection.response result = connection.result + if result['referrals'] and original_auto_referrals: # if rererrals are returned start over the loop with a new connection to the referral + if not original_connection: + original_connection = connection + _, connection, cachekey = connection.strategy.create_referral_connection(result['referrals']) # change connection to a valid referrals + continue + responses.extend(response) try: cookie = result['controls']['1.2.840.113556.1.4.319']['value']['cookie'] except KeyError: cookie = None - if result and result['result'] not in DO_NOT_RAISE_EXCEPTIONS: + if connection.raise_exceptions and result and result['result'] not in DO_NOT_RAISE_EXCEPTIONS: if log_enabled(PROTOCOL): log(PROTOCOL, 'paged search operation result <%s> for <%s>', result, connection) if result['result'] == RESULT_SIZE_LIMIT_EXCEEDED: @@ -86,6 +96,14 @@ def paged_search_generator(connection, while responses: yield responses.pop() + if original_connection: + connection = original_connection + if connection.use_referral_cache and cachekey: + connection.strategy.referral_cache[cachekey] = connection + else: + connection.unbind() + + connection.auto_referrals = original_auto_referrals connection.response = None diff --git a/server/www/packages/packages-linux/x64/ldap3/extend/standard/PersistentSearch.py b/server/www/packages/packages-linux/x64/ldap3/extend/standard/PersistentSearch.py index 62286e1..1c73d4e 100644 --- a/server/www/packages/packages-linux/x64/ldap3/extend/standard/PersistentSearch.py +++ b/server/www/packages/packages-linux/x64/ldap3/extend/standard/PersistentSearch.py @@ -80,7 +80,8 @@ class PersistentSearch(object): else: self.controls = controls - self.controls.append(persistent_search_control(events_type, changes_only, notifications)) + if events_type and changes_only and notifications: + self.controls.append(persistent_search_control(events_type, changes_only, notifications)) self.start() def start(self): @@ -101,9 +102,10 @@ class PersistentSearch(object): controls=self.controls) self.connection.strategy.persistent_search_message_id = self.message_id - def stop(self): + def stop(self, unbind=True): self.connection.abandon(self.message_id) - self.connection.unbind() + if unbind: + self.connection.unbind() if self.message_id in self.connection.strategy._responses: del self.connection.strategy._responses[self.message_id] if hasattr(self.connection.strategy, '_requests') and self.message_id in self.connection.strategy._requests: # asynchronous strategy has a dict of request that could be returned by get_response() @@ -111,11 +113,25 @@ class PersistentSearch(object): self.connection.strategy.persistent_search_message_id = None self.message_id = None - def next(self): + def next(self, block=False, timeout=None): if not self.connection.strategy.streaming and not self.connection.strategy.callback: try: - return self.connection.strategy.events.get_nowait() + return self.connection.strategy.events.get(block, timeout) except Empty: return None raise LDAPExtensionError('Persistent search is not accumulating events in queue') + + def funnel(self, block=False, timeout=None): + esci = False + while not esci: + try: + entry = self.connection.strategy.events.get(block, timeout) + except Empty: + yield None + if entry['type'] == 'searchResEntry': + yield entry + else: + esci = True + + yield entry diff --git a/server/www/packages/packages-linux/x64/ldap3/extend/standard/modifyPassword.py b/server/www/packages/packages-linux/x64/ldap3/extend/standard/modifyPassword.py index 167816e..0ec4fd3 100644 --- a/server/www/packages/packages-linux/x64/ldap3/extend/standard/modifyPassword.py +++ b/server/www/packages/packages-linux/x64/ldap3/extend/standard/modifyPassword.py @@ -5,7 +5,7 @@ # # Author: Giovanni Cannata # -# Copyright 2014 - 2018 Giovanni Cannata +# Copyright 2014 - 2019 Giovanni Cannata # # This file is part of ldap3. # @@ -67,6 +67,6 @@ class ModifyPassword(ExtendedOperation): self.result[self.response_attribute] = True else: # change was not successful, raises exception if raise_exception = True in connection or returns the operation result, error code is in result['result'] self.result[self.response_attribute] = False - if not self.connection.raise_exceptions: + if self.connection.raise_exceptions: from ...core.exceptions import LDAPOperationResult raise LDAPOperationResult(result=self.result['result'], description=self.result['description'], dn=self.result['dn'], message=self.result['message'], response_type=self.result['type']) diff --git a/server/www/packages/packages-linux/x64/ldap3/extend/standard/whoAmI.py b/server/www/packages/packages-linux/x64/ldap3/extend/standard/whoAmI.py index 0eda5c4..cb9235e 100644 --- a/server/www/packages/packages-linux/x64/ldap3/extend/standard/whoAmI.py +++ b/server/www/packages/packages-linux/x64/ldap3/extend/standard/whoAmI.py @@ -5,7 +5,7 @@ # # Author: Giovanni Cannata # -# Copyright 2014 - 2018 Giovanni Cannata +# Copyright 2014 - 2019 Giovanni Cannata # # This file is part of ldap3. # @@ -24,11 +24,9 @@ # If not, see . # implements RFC4532 -from pyasn1.type.univ import NoValue - from ...extend.operation import ExtendedOperation from ...utils.conv import to_unicode -from ...protocol.rfc4511 import OctetString + class WhoAmI(ExtendedOperation): def config(self): diff --git a/server/www/packages/packages-linux/x64/ldap3/operation/bind.py b/server/www/packages/packages-linux/x64/ldap3/operation/bind.py index 0eecc4e..78eeca8 100644 --- a/server/www/packages/packages-linux/x64/ldap3/operation/bind.py +++ b/server/www/packages/packages-linux/x64/ldap3/operation/bind.py @@ -122,7 +122,7 @@ def bind_response_to_dict(response): 'description': ResultCode().getNamedValues().getName(response['resultCode']), 'dn': str(response['matchedDN']), 'message': str(response['diagnosticMessage']), - 'referrals': referrals_to_list(response['referral']), + 'referrals': referrals_to_list(response['referral']) if response['referral'] is not None and response['referral'].hasValue() else [], 'saslCreds': bytes(response['serverSaslCreds']) if response['serverSaslCreds'] is not None and response['serverSaslCreds'].hasValue() else None} diff --git a/server/www/packages/packages-linux/x64/ldap3/operation/search.py b/server/www/packages/packages-linux/x64/ldap3/operation/search.py index 7cf2fb3..70088d6 100644 --- a/server/www/packages/packages-linux/x64/ldap3/operation/search.py +++ b/server/www/packages/packages-linux/x64/ldap3/operation/search.py @@ -525,10 +525,11 @@ def search_result_entry_response_to_dict(response, schema, custom_formatter, che entry = dict() # entry['dn'] = str(response['object']) if response['object']: - entry['raw_dn'] = to_raw(response['object']) if isinstance(response['object'], STRING_TYPES): # mock strategies return string not a PyAsn1 object + entry['raw_dn'] = to_raw(response['object']) entry['dn'] = to_unicode(response['object']) else: + entry['raw_dn'] = str(response['object']) entry['dn'] = to_unicode(bytes(response['object']), from_server=True) else: entry['raw_dn'] = b'' @@ -555,6 +556,8 @@ def search_result_done_response_to_dict(response): result['controls'][control[0]] = control[1] return result + + def search_result_reference_response_to_dict(response): return {'uri': search_refs_to_list(response)} diff --git a/server/www/packages/packages-linux/x64/ldap3/protocol/convert.py b/server/www/packages/packages-linux/x64/ldap3/protocol/convert.py index 319f36d..e3a94f3 100644 --- a/server/www/packages/packages-linux/x64/ldap3/protocol/convert.py +++ b/server/www/packages/packages-linux/x64/ldap3/protocol/convert.py @@ -37,6 +37,7 @@ def attribute_to_dict(attribute): except PyAsn1Error: # invalid encoding, return bytes value return {'type': str(attribute['type']), 'values': [bytes(val) for val in attribute['vals']]} + def attributes_to_dict(attributes): attributes_dict = dict() for attribute in attributes: @@ -46,7 +47,10 @@ def attributes_to_dict(attributes): def referrals_to_list(referrals): - return [str(referral) for referral in referrals if referral] if referrals else None + if isinstance(referrals, list): + return [str(referral) for referral in referrals if referral] if referrals else None + else: + return [str(referral) for referral in referrals if referral] if referrals is not None and referrals.hasValue() else None def search_refs_to_list(search_refs): @@ -93,6 +97,7 @@ def ava_to_dict(ava): except Exception: return {'attribute': str(ava['attributeDesc']), 'value': bytes(ava['assertionValue'])} + def substring_to_dict(substring): return {'initial': substring['initial'] if substring['initial'] else '', 'any': [middle for middle in substring['any']] if substring['any'] else '', 'final': substring['final'] if substring['final'] else ''} diff --git a/server/www/packages/packages-linux/x64/ldap3/protocol/formatters/formatters.py b/server/www/packages/packages-linux/x64/ldap3/protocol/formatters/formatters.py index 36cd9c8..f7bfe0c 100644 --- a/server/www/packages/packages-linux/x64/ldap3/protocol/formatters/formatters.py +++ b/server/www/packages/packages-linux/x64/ldap3/protocol/formatters/formatters.py @@ -1,407 +1,436 @@ -""" -""" - -# Created on 2014.10.28 -# -# Author: Giovanni Cannata -# -# Copyright 2014 - 2018 Giovanni Cannata -# -# This file is part of ldap3. -# -# ldap3 is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# ldap3 is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with ldap3 in the COPYING and COPYING.LESSER files. -# If not, see . - -import re - -from binascii import hexlify -from uuid import UUID -from datetime import datetime, timedelta -from ...utils.conv import to_unicode - -from ...core.timezone import OffsetTzInfo - -def format_unicode(raw_value): - try: - if str is not bytes: # Python 3 - return str(raw_value, 'utf-8', errors='strict') - else: # Python 2 - return unicode(raw_value, 'utf-8', errors='strict') - except (TypeError, UnicodeDecodeError): - pass - - return raw_value - - -def format_integer(raw_value): - try: - return int(raw_value) - except (TypeError, ValueError): # expected exceptions - pass - except Exception: # any other exception should be investigated, anyway the formatter return the raw_value - pass - - return raw_value - - -def format_binary(raw_value): - try: - return bytes(raw_value) - except TypeError: # expected exceptions - pass - except Exception: # any other exception should be investigated, anyway the formatter return the raw_value - pass - - return raw_value - - -def format_uuid(raw_value): - try: - return str(UUID(bytes=raw_value)) - except (TypeError, ValueError): - return format_unicode(raw_value) - except Exception: # any other exception should be investigated, anyway the formatter return the raw_value - pass - - return raw_value - - -def format_uuid_le(raw_value): - try: - return '{' + str(UUID(bytes_le=raw_value)) + '}' - except (TypeError, ValueError): - return format_unicode(raw_value) - except Exception: # any other exception should be investigated, anyway the formatter return the raw_value - pass - - return raw_value - - -def format_boolean(raw_value): - if raw_value in [b'TRUE', b'true', b'True']: - return True - if raw_value in [b'FALSE', b'false', b'False']: - return False - - return raw_value - - -def format_ad_timestamp(raw_value): - """ - Active Directory stores date/time values as the number of 100-nanosecond intervals - that have elapsed since the 0 hour on January 1, 1601 till the date/time that is being stored. - The time is always stored in Greenwich Mean Time (GMT) in the Active Directory. - """ - if raw_value == b'9223372036854775807': # max value to be stored in a 64 bit signed int - return datetime.max # returns datetime.datetime(9999, 12, 31, 23, 59, 59, 999999) - try: - timestamp = int(raw_value) - if timestamp < 0: # ad timestamp cannot be negative - return raw_value - except Exception: - return raw_value - - try: - return datetime.fromtimestamp(timestamp / 10000000.0 - 11644473600, tz=OffsetTzInfo(0, 'UTC')) # forces true division in python 2 - except (OSError, OverflowError, ValueError): # on Windows backwards timestamps are not allowed - try: - unix_epoch = datetime.fromtimestamp(0, tz=OffsetTzInfo(0, 'UTC')) - diff_seconds = timedelta(seconds=timestamp/10000000.0 - 11644473600) - return unix_epoch + diff_seconds - except Exception: - pass - except Exception: - pass - - return raw_value - - -try: # uses regular expressions and the timezone class (python3.2 and later) - from datetime import timezone - time_format = re.compile( - r''' - ^ - (?P[0-9]{4}) - (?P0[1-9]|1[0-2]) - (?P0[1-9]|[12][0-9]|3[01]) - (?P[01][0-9]|2[0-3]) - (?: - (?P[0-5][0-9]) - (?P[0-5][0-9]|60)? - )? - (?: - [.,] - (?P[0-9]+) - )? - (?: - Z - | - (?: - (?P[+-]) - (?P[01][0-9]|2[0-3]) - (?P[0-5][0-9])? - ) - ) - $ - ''', - re.VERBOSE - ) - - def format_time(raw_value): - try: - match = time_format.fullmatch(to_unicode(raw_value)) - if match is None: - return raw_value - matches = match.groupdict() - - offset = timedelta( - hours=int(matches['OffHour'] or 0), - minutes=int(matches['OffMinute'] or 0) - ) - - if matches['Offset'] == '-': - offset *= -1 - - # Python does not support leap second in datetime (!) - if matches['Second'] == '60': - matches['Second'] = '59' - - # According to RFC, fraction may be applied to an Hour/Minute (!) - fraction = float('0.' + (matches['Fraction'] or '0')) - - if matches['Minute'] is None: - fraction *= 60 - minute = int(fraction) - fraction -= minute - else: - minute = int(matches['Minute']) - - if matches['Second'] is None: - fraction *= 60 - second = int(fraction) - fraction -= second - else: - second = int(matches['Second']) - - microseconds = int(fraction * 1000000) - - return datetime( - int(matches['Year']), - int(matches['Month']), - int(matches['Day']), - int(matches['Hour']), - minute, - second, - microseconds, - timezone(offset), - ) - except Exception: # exceptions should be investigated, anyway the formatter return the raw_value - pass - return raw_value - -except ImportError: - def format_time(raw_value): - """ - From RFC4517: - A value of the Generalized Time syntax is a character string - representing a date and time. The LDAP-specific encoding of a value - of this syntax is a restriction of the format defined in [ISO8601], - and is described by the following ABNF: - - GeneralizedTime = century year month day hour - [ minute [ second / leap-second ] ] - [ fraction ] - g-time-zone - - century = 2(%x30-39) ; "00" to "99" - year = 2(%x30-39) ; "00" to "99" - month = ( %x30 %x31-39 ) ; "01" (January) to "09" - / ( %x31 %x30-32 ) ; "10" to "12" - day = ( %x30 %x31-39 ) ; "01" to "09" - / ( %x31-32 %x30-39 ) ; "10" to "29" - / ( %x33 %x30-31 ) ; "30" to "31" - hour = ( %x30-31 %x30-39 ) / ( %x32 %x30-33 ) ; "00" to "23" - minute = %x30-35 %x30-39 ; "00" to "59" - second = ( %x30-35 %x30-39 ) ; "00" to "59" - leap-second = ( %x36 %x30 ) ; "60" - fraction = ( DOT / COMMA ) 1*(%x30-39) - g-time-zone = %x5A ; "Z" - / g-differential - g-differential = ( MINUS / PLUS ) hour [ minute ] - MINUS = %x2D ; minus sign ("-") - """ - - if len(raw_value) < 10 or not all((c in b'0123456789+-,.Z' for c in raw_value)) or (b'Z' in raw_value and not raw_value.endswith(b'Z')): # first ten characters are mandatory and must be numeric or timezone or fraction - return raw_value - - # sets position for fixed values - year = int(raw_value[0: 4]) - month = int(raw_value[4: 6]) - day = int(raw_value[6: 8]) - hour = int(raw_value[8: 10]) - minute = 0 - second = 0 - microsecond = 0 - - remain = raw_value[10:] - if remain and remain.endswith(b'Z'): # uppercase 'Z' - sep = b'Z' - elif b'+' in remain: # timezone can be specified with +hh[mm] or -hh[mm] - sep = b'+' - elif b'-' in remain: - sep = b'-' - else: # timezone not specified - return raw_value - - time, _, offset = remain.partition(sep) - - if time and (b'.' in time or b',' in time): - # fraction time - if time[0] in b',.': - minute = 6 * int(time[1] if str is bytes else chr(time[1])) # Python 2 / Python 3 - elif time[2] in b',.': - minute = int(raw_value[10: 12]) - second = 6 * int(time[3] if str is bytes else chr(time[3])) # Python 2 / Python 3 - elif time[4] in b',.': - minute = int(raw_value[10: 12]) - second = int(raw_value[12: 14]) - microsecond = 100000 * int(time[5] if str is bytes else chr(time[5])) # Python 2 / Python 3 - elif len(time) == 2: # mmZ format - minute = int(raw_value[10: 12]) - elif len(time) == 0: # Z format - pass - elif len(time) == 4: # mmssZ - minute = int(raw_value[10: 12]) - second = int(raw_value[12: 14]) - else: - return raw_value - - if sep == b'Z': # UTC - timezone = OffsetTzInfo(0, 'UTC') - else: # build timezone - try: - if len(offset) == 2: - timezone_hour = int(offset[:2]) - timezone_minute = 0 - elif len(offset) == 4: - timezone_hour = int(offset[:2]) - timezone_minute = int(offset[2:4]) - else: # malformed timezone - raise ValueError - except ValueError: - return raw_value - if timezone_hour > 23 or timezone_minute > 59: # invalid timezone - return raw_value - - if str is not bytes: # Python 3 - timezone = OffsetTzInfo((timezone_hour * 60 + timezone_minute) * (1 if sep == b'+' else -1), 'UTC' + str(sep + offset, encoding='utf-8')) - else: # Python 2 - timezone = OffsetTzInfo((timezone_hour * 60 + timezone_minute) * (1 if sep == b'+' else -1), unicode('UTC' + sep + offset, encoding='utf-8')) - - try: - return datetime(year=year, - month=month, - day=day, - hour=hour, - minute=minute, - second=second, - microsecond=microsecond, - tzinfo=timezone) - except (TypeError, ValueError): - pass - - return raw_value - - -def format_time_with_0_year(raw_value): - try: - if raw_value.startswith(b'0000'): - return raw_value - except Exception: - try: - if raw_value.startswith('0000'): - return raw_value - except Exception: - pass - - return format_time(raw_value) - - -def format_sid(raw_value): - """ - SID= "S-1-" IdentifierAuthority 1*SubAuthority - IdentifierAuthority= IdentifierAuthorityDec / IdentifierAuthorityHex - ; If the identifier authority is < 2^32, the - ; identifier authority is represented as a decimal - ; number - ; If the identifier authority is >= 2^32, - ; the identifier authority is represented in - ; hexadecimal - IdentifierAuthorityDec = 1*10DIGIT - ; IdentifierAuthorityDec, top level authority of a - ; security identifier is represented as a decimal number - IdentifierAuthorityHex = "0x" 12HEXDIG - ; IdentifierAuthorityHex, the top-level authority of a - ; security identifier is represented as a hexadecimal number - SubAuthority= "-" 1*10DIGIT - ; Sub-Authority is always represented as a decimal number - ; No leading "0" characters are allowed when IdentifierAuthority - ; or SubAuthority is represented as a decimal number - ; All hexadecimal digits must be output in string format, - ; pre-pended by "0x" - - Revision (1 byte): An 8-bit unsigned integer that specifies the revision level of the SID. This value MUST be set to 0x01. - SubAuthorityCount (1 byte): An 8-bit unsigned integer that specifies the number of elements in the SubAuthority array. The maximum number of elements allowed is 15. - IdentifierAuthority (6 bytes): A SID_IDENTIFIER_AUTHORITY structure that indicates the authority under which the SID was created. It describes the entity that created the SID. The Identifier Authority value {0,0,0,0,0,5} denotes SIDs created by the NT SID authority. - SubAuthority (variable): A variable length array of unsigned 32-bit integers that uniquely identifies a principal relative to the IdentifierAuthority. Its length is determined by SubAuthorityCount. - """ - try: - if raw_value.startswith(b'S-1-'): - return raw_value - except Exception: - try: - if raw_value.startswith('S-1-'): - return raw_value - except Exception: - pass - try: - if str is not bytes: # Python 3 - revision = int(raw_value[0]) - sub_authority_count = int(raw_value[1]) - identifier_authority = int.from_bytes(raw_value[2:8], byteorder='big') - if identifier_authority >= 4294967296: # 2 ^ 32 - identifier_authority = hex(identifier_authority) - - sub_authority = '' - i = 0 - while i < sub_authority_count: - sub_authority += '-' + str(int.from_bytes(raw_value[8 + (i * 4): 12 + (i * 4)], byteorder='little')) # little endian - i += 1 - else: # Python 2 - revision = int(ord(raw_value[0])) - sub_authority_count = int(ord(raw_value[1])) - identifier_authority = int(hexlify(raw_value[2:8]), 16) - if identifier_authority >= 4294967296: # 2 ^ 32 - identifier_authority = hex(identifier_authority) - - sub_authority = '' - i = 0 - while i < sub_authority_count: - sub_authority += '-' + str(int(hexlify(raw_value[11 + (i * 4): 7 + (i * 4): -1]), 16)) # little endian - i += 1 - return 'S-' + str(revision) + '-' + str(identifier_authority) + sub_authority - except Exception: # any exception should be investigated, anyway the formatter return the raw_value - pass - - return raw_value +""" +""" + +# Created on 2014.10.28 +# +# Author: Giovanni Cannata +# +# Copyright 2014 - 2019 Giovanni Cannata +# +# This file is part of ldap3. +# +# ldap3 is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ldap3 is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with ldap3 in the COPYING and COPYING.LESSER files. +# If not, see . + +import re + +from binascii import hexlify +from uuid import UUID +from datetime import datetime, timedelta +from ...utils.conv import to_unicode + +from ...core.timezone import OffsetTzInfo + + +def format_unicode(raw_value): + try: + if str is not bytes: # Python 3 + return str(raw_value, 'utf-8', errors='strict') + else: # Python 2 + return unicode(raw_value, 'utf-8', errors='strict') + except (TypeError, UnicodeDecodeError): + pass + + return raw_value + + +def format_integer(raw_value): + try: + return int(raw_value) + except (TypeError, ValueError): # expected exceptions + pass + except Exception: # any other exception should be investigated, anyway the formatter return the raw_value + pass + + return raw_value + + +def format_binary(raw_value): + try: + return bytes(raw_value) + except TypeError: # expected exceptions + pass + except Exception: # any other exception should be investigated, anyway the formatter return the raw_value + pass + + return raw_value + + +def format_uuid(raw_value): + try: + return str(UUID(bytes=raw_value)) + except (TypeError, ValueError): + return format_unicode(raw_value) + except Exception: # any other exception should be investigated, anyway the formatter return the raw_value + pass + + return raw_value + + +def format_uuid_le(raw_value): + try: + return '{' + str(UUID(bytes_le=raw_value)) + '}' + except (TypeError, ValueError): + return format_unicode(raw_value) + except Exception: # any other exception should be investigated, anyway the formatter return the raw_value + pass + + return raw_value + + +def format_boolean(raw_value): + if raw_value in [b'TRUE', b'true', b'True']: + return True + if raw_value in [b'FALSE', b'false', b'False']: + return False + + return raw_value + + +def format_ad_timestamp(raw_value): + """ + Active Directory stores date/time values as the number of 100-nanosecond intervals + that have elapsed since the 0 hour on January 1, 1601 till the date/time that is being stored. + The time is always stored in Greenwich Mean Time (GMT) in the Active Directory. + """ + utc_timezone = OffsetTzInfo(0, 'UTC') + if raw_value == b'9223372036854775807': # max value to be stored in a 64 bit signed int + return datetime.max.replace(tzinfo=utc_timezone) # returns datetime.datetime(9999, 12, 31, 23, 59, 59, 999999, tzinfo=OffsetTzInfo(offset=0, name='UTC')) + try: + timestamp = int(raw_value) + if timestamp < 0: # ad timestamp cannot be negative + timestamp = timestamp * -1 + except Exception: + return raw_value + + try: + return datetime.fromtimestamp(timestamp / 10000000.0 - 11644473600, + tz=utc_timezone) # forces true division in python 2 + except (OSError, OverflowError, ValueError): # on Windows backwards timestamps are not allowed + try: + unix_epoch = datetime.fromtimestamp(0, tz=utc_timezone) + diff_seconds = timedelta(seconds=timestamp / 10000000.0 - 11644473600) + return unix_epoch + diff_seconds + except Exception: + pass + except Exception: + pass + + return raw_value + + +try: # uses regular expressions and the timezone class (python3.2 and later) + from datetime import timezone + + time_format = re.compile( + r''' + ^ + (?P[0-9]{4}) + (?P0[1-9]|1[0-2]) + (?P0[1-9]|[12][0-9]|3[01]) + (?P[01][0-9]|2[0-3]) + (?: + (?P[0-5][0-9]) + (?P[0-5][0-9]|60)? + )? + (?: + [.,] + (?P[0-9]+) + )? + (?: + Z + | + (?: + (?P[+-]) + (?P[01][0-9]|2[0-3]) + (?P[0-5][0-9])? + ) + ) + $ + ''', + re.VERBOSE + ) + + + def format_time(raw_value): + try: + match = time_format.fullmatch(to_unicode(raw_value)) + if match is None: + return raw_value + matches = match.groupdict() + + offset = timedelta( + hours=int(matches['OffHour'] or 0), + minutes=int(matches['OffMinute'] or 0) + ) + + if matches['Offset'] == '-': + offset *= -1 + + # Python does not support leap second in datetime (!) + if matches['Second'] == '60': + matches['Second'] = '59' + + # According to RFC, fraction may be applied to an Hour/Minute (!) + fraction = float('0.' + (matches['Fraction'] or '0')) + + if matches['Minute'] is None: + fraction *= 60 + minute = int(fraction) + fraction -= minute + else: + minute = int(matches['Minute']) + + if matches['Second'] is None: + fraction *= 60 + second = int(fraction) + fraction -= second + else: + second = int(matches['Second']) + + microseconds = int(fraction * 1000000) + + return datetime( + int(matches['Year']), + int(matches['Month']), + int(matches['Day']), + int(matches['Hour']), + minute, + second, + microseconds, + timezone(offset), + ) + except Exception: # exceptions should be investigated, anyway the formatter return the raw_value + pass + return raw_value + +except ImportError: + def format_time(raw_value): + """ + From RFC4517: + A value of the Generalized Time syntax is a character string + representing a date and time. The LDAP-specific encoding of a value + of this syntax is a restriction of the format defined in [ISO8601], + and is described by the following ABNF: + + GeneralizedTime = century year month day hour + [ minute [ second / leap-second ] ] + [ fraction ] + g-time-zone + + century = 2(%x30-39) ; "00" to "99" + year = 2(%x30-39) ; "00" to "99" + month = ( %x30 %x31-39 ) ; "01" (January) to "09" + / ( %x31 %x30-32 ) ; "10" to "12" + day = ( %x30 %x31-39 ) ; "01" to "09" + / ( %x31-32 %x30-39 ) ; "10" to "29" + / ( %x33 %x30-31 ) ; "30" to "31" + hour = ( %x30-31 %x30-39 ) / ( %x32 %x30-33 ) ; "00" to "23" + minute = %x30-35 %x30-39 ; "00" to "59" + second = ( %x30-35 %x30-39 ) ; "00" to "59" + leap-second = ( %x36 %x30 ) ; "60" + fraction = ( DOT / COMMA ) 1*(%x30-39) + g-time-zone = %x5A ; "Z" + / g-differential + g-differential = ( MINUS / PLUS ) hour [ minute ] + MINUS = %x2D ; minus sign ("-") + """ + + if len(raw_value) < 10 or not all((c in b'0123456789+-,.Z' for c in raw_value)) or ( + b'Z' in raw_value and not raw_value.endswith( + b'Z')): # first ten characters are mandatory and must be numeric or timezone or fraction + return raw_value + + # sets position for fixed values + year = int(raw_value[0: 4]) + month = int(raw_value[4: 6]) + day = int(raw_value[6: 8]) + hour = int(raw_value[8: 10]) + minute = 0 + second = 0 + microsecond = 0 + + remain = raw_value[10:] + if remain and remain.endswith(b'Z'): # uppercase 'Z' + sep = b'Z' + elif b'+' in remain: # timezone can be specified with +hh[mm] or -hh[mm] + sep = b'+' + elif b'-' in remain: + sep = b'-' + else: # timezone not specified + return raw_value + + time, _, offset = remain.partition(sep) + + if time and (b'.' in time or b',' in time): + # fraction time + if time[0] in b',.': + minute = 6 * int(time[1] if str is bytes else chr(time[1])) # Python 2 / Python 3 + elif time[2] in b',.': + minute = int(raw_value[10: 12]) + second = 6 * int(time[3] if str is bytes else chr(time[3])) # Python 2 / Python 3 + elif time[4] in b',.': + minute = int(raw_value[10: 12]) + second = int(raw_value[12: 14]) + microsecond = 100000 * int(time[5] if str is bytes else chr(time[5])) # Python 2 / Python 3 + elif len(time) == 2: # mmZ format + minute = int(raw_value[10: 12]) + elif len(time) == 0: # Z format + pass + elif len(time) == 4: # mmssZ + minute = int(raw_value[10: 12]) + second = int(raw_value[12: 14]) + else: + return raw_value + + if sep == b'Z': # UTC + timezone = OffsetTzInfo(0, 'UTC') + else: # build timezone + try: + if len(offset) == 2: + timezone_hour = int(offset[:2]) + timezone_minute = 0 + elif len(offset) == 4: + timezone_hour = int(offset[:2]) + timezone_minute = int(offset[2:4]) + else: # malformed timezone + raise ValueError + except ValueError: + return raw_value + if timezone_hour > 23 or timezone_minute > 59: # invalid timezone + return raw_value + + if str is not bytes: # Python 3 + timezone = OffsetTzInfo((timezone_hour * 60 + timezone_minute) * (1 if sep == b'+' else -1), + 'UTC' + str(sep + offset, encoding='utf-8')) + else: # Python 2 + timezone = OffsetTzInfo((timezone_hour * 60 + timezone_minute) * (1 if sep == b'+' else -1), + unicode('UTC' + sep + offset, encoding='utf-8')) + + try: + return datetime(year=year, + month=month, + day=day, + hour=hour, + minute=minute, + second=second, + microsecond=microsecond, + tzinfo=timezone) + except (TypeError, ValueError): + pass + + return raw_value + + +def format_ad_timedelta(raw_value): + """ + Convert a negative filetime value to a timedelta. + """ + # Active Directory stores attributes like "minPwdAge" as a negative + # "filetime" timestamp, which is the number of 100-nanosecond intervals that + # have elapsed since the 0 hour on January 1, 1601. + # + # Handle the minimum value that can be stored in a 64 bit signed integer. + # See https://docs.microsoft.com/en-us/dotnet/api/system.int64.minvalue + # In attributes like "maxPwdAge", this signifies never. + if raw_value == b'-9223372036854775808': + return timedelta.max + # We can reuse format_ad_timestamp to get a datetime object from the + # timestamp. Afterwards, we can subtract a datetime representing 0 hour on + # January 1, 1601 from the returned datetime to get the timedelta. + return format_ad_timestamp(raw_value) - format_ad_timestamp(0) + + +def format_time_with_0_year(raw_value): + try: + if raw_value.startswith(b'0000'): + return raw_value + except Exception: + try: + if raw_value.startswith('0000'): + return raw_value + except Exception: + pass + + return format_time(raw_value) + + +def format_sid(raw_value): + """ + SID= "S-1-" IdentifierAuthority 1*SubAuthority + IdentifierAuthority= IdentifierAuthorityDec / IdentifierAuthorityHex + ; If the identifier authority is < 2^32, the + ; identifier authority is represented as a decimal + ; number + ; If the identifier authority is >= 2^32, + ; the identifier authority is represented in + ; hexadecimal + IdentifierAuthorityDec = 1*10DIGIT + ; IdentifierAuthorityDec, top level authority of a + ; security identifier is represented as a decimal number + IdentifierAuthorityHex = "0x" 12HEXDIG + ; IdentifierAuthorityHex, the top-level authority of a + ; security identifier is represented as a hexadecimal number + SubAuthority= "-" 1*10DIGIT + ; Sub-Authority is always represented as a decimal number + ; No leading "0" characters are allowed when IdentifierAuthority + ; or SubAuthority is represented as a decimal number + ; All hexadecimal digits must be output in string format, + ; pre-pended by "0x" + + Revision (1 byte): An 8-bit unsigned integer that specifies the revision level of the SID. This value MUST be set to 0x01. + SubAuthorityCount (1 byte): An 8-bit unsigned integer that specifies the number of elements in the SubAuthority array. The maximum number of elements allowed is 15. + IdentifierAuthority (6 bytes): A SID_IDENTIFIER_AUTHORITY structure that indicates the authority under which the SID was created. It describes the entity that created the SID. The Identifier Authority value {0,0,0,0,0,5} denotes SIDs created by the NT SID authority. + SubAuthority (variable): A variable length array of unsigned 32-bit integers that uniquely identifies a principal relative to the IdentifierAuthority. Its length is determined by SubAuthorityCount. + """ + try: + if raw_value.startswith(b'S-1-'): + return raw_value + except Exception: + try: + if raw_value.startswith('S-1-'): + return raw_value + except Exception: + pass + try: + if str is not bytes: # Python 3 + revision = int(raw_value[0]) + sub_authority_count = int(raw_value[1]) + identifier_authority = int.from_bytes(raw_value[2:8], byteorder='big') + if identifier_authority >= 4294967296: # 2 ^ 32 + identifier_authority = hex(identifier_authority) + + sub_authority = '' + i = 0 + while i < sub_authority_count: + sub_authority += '-' + str( + int.from_bytes(raw_value[8 + (i * 4): 12 + (i * 4)], byteorder='little')) # little endian + i += 1 + else: # Python 2 + revision = int(ord(raw_value[0])) + sub_authority_count = int(ord(raw_value[1])) + identifier_authority = int(hexlify(raw_value[2:8]), 16) + if identifier_authority >= 4294967296: # 2 ^ 32 + identifier_authority = hex(identifier_authority) + + sub_authority = '' + i = 0 + while i < sub_authority_count: + sub_authority += '-' + str(int(hexlify(raw_value[11 + (i * 4): 7 + (i * 4): -1]), 16)) # little endian + i += 1 + return 'S-' + str(revision) + '-' + str(identifier_authority) + sub_authority + except Exception: # any exception should be investigated, anyway the formatter return the raw_value + pass + + return raw_value diff --git a/server/www/packages/packages-linux/x64/ldap3/protocol/formatters/standard.py b/server/www/packages/packages-linux/x64/ldap3/protocol/formatters/standard.py index 77f7b2e..51187f1 100644 --- a/server/www/packages/packages-linux/x64/ldap3/protocol/formatters/standard.py +++ b/server/www/packages/packages-linux/x64/ldap3/protocol/formatters/standard.py @@ -1,232 +1,238 @@ -""" -""" - -# Created on 2014.10.28 -# -# Author: Giovanni Cannata -# -# Copyright 2014 - 2018 Giovanni Cannata -# -# This file is part of ldap3. -# -# ldap3 is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# ldap3 is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with ldap3 in the COPYING and COPYING.LESSER files. -# If not, see . - -from ... import SEQUENCE_TYPES -from .formatters import format_ad_timestamp, format_binary, format_boolean,\ - format_integer, format_sid, format_time, format_unicode, format_uuid, format_uuid_le, format_time_with_0_year -from .validators import validate_integer, validate_time, always_valid,\ - validate_generic_single_value, validate_boolean, validate_ad_timestamp, validate_sid,\ - validate_uuid_le, validate_uuid, validate_zero_and_minus_one_and_positive_int, validate_guid, validate_time_with_0_year - -# for each syntax can be specified a format function and a input validation function - -standard_formatter = { - '1.2.840.113556.1.4.903': (format_binary, None), # Object (DN-binary) - Microsoft - '1.2.840.113556.1.4.904': (format_unicode, None), # Object (DN-string) - Microsoft - '1.2.840.113556.1.4.905': (format_unicode, None), # String (Teletex) - Microsoft - '1.2.840.113556.1.4.906': (format_integer, validate_integer), # Large integer - Microsoft - '1.2.840.113556.1.4.907': (format_binary, None), # String (NT-sec-desc) - Microsoft - '1.2.840.113556.1.4.1221': (format_binary, None), # Object (OR-name) - Microsoft - '1.2.840.113556.1.4.1362': (format_unicode, None), # String (Case) - Microsoft - '1.3.6.1.4.1.1466.115.121.1.1': (format_binary, None), # ACI item [OBSOLETE] - '1.3.6.1.4.1.1466.115.121.1.2': (format_binary, None), # Access point [OBSOLETE] - '1.3.6.1.4.1.1466.115.121.1.3': (format_unicode, None), # Attribute type description - '1.3.6.1.4.1.1466.115.121.1.4': (format_binary, None), # Audio [OBSOLETE] - '1.3.6.1.4.1.1466.115.121.1.5': (format_binary, None), # Binary [OBSOLETE] - '1.3.6.1.4.1.1466.115.121.1.6': (format_unicode, None), # Bit String - '1.3.6.1.4.1.1466.115.121.1.7': (format_boolean, validate_boolean), # Boolean - '1.3.6.1.4.1.1466.115.121.1.8': (format_binary, None), # Certificate [OBSOLETE] - '1.3.6.1.4.1.1466.115.121.1.9': (format_binary, None), # Certificate List [OBSOLETE] - '1.3.6.1.4.1.1466.115.121.1.10': (format_binary, None), # Certificate Pair [OBSOLETE] - '1.3.6.1.4.1.1466.115.121.1.11': (format_unicode, None), # Country String - '1.3.6.1.4.1.1466.115.121.1.12': (format_unicode, None), # Distinguished name (DN) - '1.3.6.1.4.1.1466.115.121.1.13': (format_binary, None), # Data Quality Syntax [OBSOLETE] - '1.3.6.1.4.1.1466.115.121.1.14': (format_unicode, None), # Delivery method - '1.3.6.1.4.1.1466.115.121.1.15': (format_unicode, None), # Directory string - '1.3.6.1.4.1.1466.115.121.1.16': (format_unicode, None), # DIT Content Rule Description - '1.3.6.1.4.1.1466.115.121.1.17': (format_unicode, None), # DIT Structure Rule Description - '1.3.6.1.4.1.1466.115.121.1.18': (format_binary, None), # DL Submit Permission [OBSOLETE] - '1.3.6.1.4.1.1466.115.121.1.19': (format_binary, None), # DSA Quality Syntax [OBSOLETE] - '1.3.6.1.4.1.1466.115.121.1.20': (format_binary, None), # DSE Type [OBSOLETE] - '1.3.6.1.4.1.1466.115.121.1.21': (format_binary, None), # Enhanced Guide - '1.3.6.1.4.1.1466.115.121.1.22': (format_unicode, None), # Facsimile Telephone Number - '1.3.6.1.4.1.1466.115.121.1.23': (format_binary, None), # Fax - '1.3.6.1.4.1.1466.115.121.1.24': (format_time, validate_time), # Generalized time - '1.3.6.1.4.1.1466.115.121.1.25': (format_binary, None), # Guide [OBSOLETE] - '1.3.6.1.4.1.1466.115.121.1.26': (format_unicode, None), # IA5 string - '1.3.6.1.4.1.1466.115.121.1.27': (format_integer, validate_integer), # Integer - '1.3.6.1.4.1.1466.115.121.1.28': (format_binary, None), # JPEG - '1.3.6.1.4.1.1466.115.121.1.29': (format_binary, None), # Master and Shadow Access Points [OBSOLETE] - '1.3.6.1.4.1.1466.115.121.1.30': (format_unicode, None), # Matching rule description - '1.3.6.1.4.1.1466.115.121.1.31': (format_unicode, None), # Matching rule use description - '1.3.6.1.4.1.1466.115.121.1.32': (format_unicode, None), # Mail Preference [OBSOLETE] - '1.3.6.1.4.1.1466.115.121.1.33': (format_unicode, None), # MHS OR Address [OBSOLETE] - '1.3.6.1.4.1.1466.115.121.1.34': (format_unicode, None), # Name and optional UID - '1.3.6.1.4.1.1466.115.121.1.35': (format_unicode, None), # Name form description - '1.3.6.1.4.1.1466.115.121.1.36': (format_unicode, None), # Numeric string - '1.3.6.1.4.1.1466.115.121.1.37': (format_unicode, None), # Object class description - '1.3.6.1.4.1.1466.115.121.1.38': (format_unicode, None), # OID - '1.3.6.1.4.1.1466.115.121.1.39': (format_unicode, None), # Other mailbox - '1.3.6.1.4.1.1466.115.121.1.40': (format_binary, None), # Octet string - '1.3.6.1.4.1.1466.115.121.1.41': (format_unicode, None), # Postal address - '1.3.6.1.4.1.1466.115.121.1.42': (format_binary, None), # Protocol Information [OBSOLETE] - '1.3.6.1.4.1.1466.115.121.1.43': (format_binary, None), # Presentation Address [OBSOLETE] - '1.3.6.1.4.1.1466.115.121.1.44': (format_unicode, None), # Printable string - '1.3.6.1.4.1.1466.115.121.1.45': (format_binary, None), # Subtree specification [OBSOLETE - '1.3.6.1.4.1.1466.115.121.1.46': (format_binary, None), # Supplier Information [OBSOLETE] - '1.3.6.1.4.1.1466.115.121.1.47': (format_binary, None), # Supplier Or Consumer [OBSOLETE] - '1.3.6.1.4.1.1466.115.121.1.48': (format_binary, None), # Supplier And Consumer [OBSOLETE] - '1.3.6.1.4.1.1466.115.121.1.49': (format_binary, None), # Supported Algorithm [OBSOLETE] - '1.3.6.1.4.1.1466.115.121.1.50': (format_unicode, None), # Telephone number - '1.3.6.1.4.1.1466.115.121.1.51': (format_unicode, None), # Teletex terminal identifier - '1.3.6.1.4.1.1466.115.121.1.52': (format_unicode, None), # Teletex number - '1.3.6.1.4.1.1466.115.121.1.53': (format_time, validate_time), # Utc time (deprecated) - '1.3.6.1.4.1.1466.115.121.1.54': (format_unicode, None), # LDAP syntax description - '1.3.6.1.4.1.1466.115.121.1.55': (format_binary, None), # Modify rights [OBSOLETE] - '1.3.6.1.4.1.1466.115.121.1.56': (format_binary, None), # LDAP Schema Definition [OBSOLETE] - '1.3.6.1.4.1.1466.115.121.1.57': (format_unicode, None), # LDAP Schema Description [OBSOLETE] - '1.3.6.1.4.1.1466.115.121.1.58': (format_unicode, None), # Substring assertion - '1.3.6.1.1.16.1': (format_uuid, validate_uuid), # UUID - '1.3.6.1.1.16.4': (format_uuid, validate_uuid), # entryUUID (RFC 4530) - '2.16.840.1.113719.1.1.4.1.501': (format_uuid, validate_guid), # GUID (Novell) - '2.16.840.1.113719.1.1.5.1.0': (format_binary, None), # Unknown (Novell) - '2.16.840.1.113719.1.1.5.1.6': (format_unicode, None), # Case Ignore List (Novell) - '2.16.840.1.113719.1.1.5.1.12': (format_binary, None), # Tagged Data (Novell) - '2.16.840.1.113719.1.1.5.1.13': (format_binary, None), # Octet List (Novell) - '2.16.840.1.113719.1.1.5.1.14': (format_unicode, None), # Tagged String (Novell) - '2.16.840.1.113719.1.1.5.1.15': (format_unicode, None), # Tagged Name And String (Novell) - '2.16.840.1.113719.1.1.5.1.16': (format_binary, None), # NDS Replica Pointer (Novell) - '2.16.840.1.113719.1.1.5.1.17': (format_unicode, None), # NDS ACL (Novell) - '2.16.840.1.113719.1.1.5.1.19': (format_time, validate_time), # NDS Timestamp (Novell) - '2.16.840.1.113719.1.1.5.1.22': (format_integer, validate_integer), # Counter (Novell) - '2.16.840.1.113719.1.1.5.1.23': (format_unicode, None), # Tagged Name (Novell) - '2.16.840.1.113719.1.1.5.1.25': (format_unicode, None), # Typed Name (Novell) - 'supportedldapversion': (format_integer, None), # supportedLdapVersion (Microsoft) - 'octetstring': (format_binary, validate_uuid_le), # octect string (Microsoft) - '1.2.840.113556.1.4.2': (format_uuid_le, validate_uuid_le), # object guid (Microsoft) - '1.2.840.113556.1.4.13': (format_ad_timestamp, validate_ad_timestamp), # builtinCreationTime (Microsoft) - '1.2.840.113556.1.4.26': (format_ad_timestamp, validate_ad_timestamp), # creationTime (Microsoft) - '1.2.840.113556.1.4.49': (format_ad_timestamp, validate_ad_timestamp), # badPasswordTime (Microsoft) - '1.2.840.113556.1.4.51': (format_ad_timestamp, validate_ad_timestamp), # lastLogoff (Microsoft) - '1.2.840.113556.1.4.52': (format_ad_timestamp, validate_ad_timestamp), # lastLogon (Microsoft) - '1.2.840.113556.1.4.96': (format_ad_timestamp, validate_zero_and_minus_one_and_positive_int), # pwdLastSet (Microsoft, can be set to -1 only) - '1.2.840.113556.1.4.146': (format_sid, validate_sid), # objectSid (Microsoft) - '1.2.840.113556.1.4.159': (format_ad_timestamp, validate_ad_timestamp), # accountExpires (Microsoft) - '1.2.840.113556.1.4.662': (format_ad_timestamp, validate_ad_timestamp), # lockoutTime (Microsoft) - '1.2.840.113556.1.4.1696': (format_ad_timestamp, validate_ad_timestamp), # lastLogonTimestamp (Microsoft) - '1.3.6.1.4.1.42.2.27.8.1.17': (format_time_with_0_year, validate_time_with_0_year) # pwdAccountLockedTime (Novell) -} - - -def find_attribute_helpers(attr_type, name, custom_formatter): - """ - Tries to format following the OIDs info and format_helper specification. - Search for attribute oid, then attribute name (can be multiple), then attribute syntax - Precedence is: - 1. attribute name - 2. attribute oid(from schema) - 3. attribute names (from oid_info) - 4. attribute syntax (from schema) - Custom formatters can be defined in Server object and have precedence over the standard_formatters - If no formatter is found the raw_value is returned as bytes. - Attributes defined as SINGLE_VALUE in schema are returned as a single object, otherwise are returned as a list of object - Formatter functions can return any kind of object - return a tuple (formatter, validator) - """ - formatter = None - if custom_formatter and isinstance(custom_formatter, dict): # if custom formatters are defined they have precedence over the standard formatters - if name in custom_formatter: # search for attribute name, as returned by the search operation - formatter = custom_formatter[name] - - if not formatter and attr_type and attr_type.oid in custom_formatter: # search for attribute oid as returned by schema - formatter = custom_formatter[attr_type.oid] - if not formatter and attr_type and attr_type.oid_info: - if isinstance(attr_type.oid_info[2], SEQUENCE_TYPES): # search for multiple names defined in oid_info - for attr_name in attr_type.oid_info[2]: - if attr_name in custom_formatter: - formatter = custom_formatter[attr_name] - break - elif attr_type.oid_info[2] in custom_formatter: # search for name defined in oid_info - formatter = custom_formatter[attr_type.oid_info[2]] - - if not formatter and attr_type and attr_type.syntax in custom_formatter: # search for syntax defined in schema - formatter = custom_formatter[attr_type.syntax] - - if not formatter and name in standard_formatter: # search for attribute name, as returned by the search operation - formatter = standard_formatter[name] - - if not formatter and attr_type and attr_type.oid in standard_formatter: # search for attribute oid as returned by schema - formatter = standard_formatter[attr_type.oid] - - if not formatter and attr_type and attr_type.oid_info: - if isinstance(attr_type.oid_info[2], SEQUENCE_TYPES): # search for multiple names defined in oid_info - for attr_name in attr_type.oid_info[2]: - if attr_name in standard_formatter: - formatter = standard_formatter[attr_name] - break - elif attr_type.oid_info[2] in standard_formatter: # search for name defined in oid_info - formatter = standard_formatter[attr_type.oid_info[2]] - if not formatter and attr_type and attr_type.syntax in standard_formatter: # search for syntax defined in schema - formatter = standard_formatter[attr_type.syntax] - - if formatter is None: - return None, None - - return formatter - - -def format_attribute_values(schema, name, values, custom_formatter): - if not values: # RFCs states that attributes must always have values, but a flaky server returns empty values too - return [] - - if not isinstance(values, SEQUENCE_TYPES): - values = [values] - - if schema and schema.attribute_types and name in schema.attribute_types: - attr_type = schema.attribute_types[name] - else: - attr_type = None - - attribute_helpers = find_attribute_helpers(attr_type, name, custom_formatter) - if not isinstance(attribute_helpers, tuple): # custom formatter - formatter = attribute_helpers - else: - formatter = format_unicode if not attribute_helpers[0] else attribute_helpers[0] - - formatted_values = [formatter(raw_value) for raw_value in values] # executes formatter - if formatted_values: - return formatted_values[0] if (attr_type and attr_type.single_value) else formatted_values - else: # RFCs states that attributes must always have values, but AD return empty values in DirSync - return [] - - -def find_attribute_validator(schema, name, custom_validator): - if schema and schema.attribute_types and name in schema.attribute_types: - attr_type = schema.attribute_types[name] - else: - attr_type = None - - attribute_helpers = find_attribute_helpers(attr_type, name, custom_validator) - if not isinstance(attribute_helpers, tuple): # custom validator - validator = attribute_helpers - else: - if not attribute_helpers[1]: - if attr_type and attr_type.single_value: - validator = validate_generic_single_value # validate only single value - else: - validator = always_valid # unknown syntax, accepts single and multi value - else: - validator = attribute_helpers[1] - return validator +""" +""" + +# Created on 2014.10.28 +# +# Author: Giovanni Cannata +# +# Copyright 2014 - 2019 Giovanni Cannata +# +# This file is part of ldap3. +# +# ldap3 is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ldap3 is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with ldap3 in the COPYING and COPYING.LESSER files. +# If not, see . + +from ... import SEQUENCE_TYPES +from .formatters import format_ad_timestamp, format_binary, format_boolean,\ + format_integer, format_sid, format_time, format_unicode, format_uuid, format_uuid_le, format_time_with_0_year,\ + format_ad_timedelta +from .validators import validate_integer, validate_time, always_valid,\ + validate_generic_single_value, validate_boolean, validate_ad_timestamp, validate_sid,\ + validate_uuid_le, validate_uuid, validate_zero_and_minus_one_and_positive_int, validate_guid, validate_time_with_0_year,\ + validate_ad_timedelta + +# for each syntax can be specified a format function and a input validation function + +standard_formatter = { + '1.2.840.113556.1.4.903': (format_binary, None), # Object (DN-binary) - Microsoft + '1.2.840.113556.1.4.904': (format_unicode, None), # Object (DN-string) - Microsoft + '1.2.840.113556.1.4.905': (format_unicode, None), # String (Teletex) - Microsoft + '1.2.840.113556.1.4.906': (format_integer, validate_integer), # Large integer - Microsoft + '1.2.840.113556.1.4.907': (format_binary, None), # String (NT-sec-desc) - Microsoft + '1.2.840.113556.1.4.1221': (format_binary, None), # Object (OR-name) - Microsoft + '1.2.840.113556.1.4.1362': (format_unicode, None), # String (Case) - Microsoft + '1.3.6.1.4.1.1466.115.121.1.1': (format_binary, None), # ACI item [OBSOLETE] + '1.3.6.1.4.1.1466.115.121.1.2': (format_binary, None), # Access point [OBSOLETE] + '1.3.6.1.4.1.1466.115.121.1.3': (format_unicode, None), # Attribute type description + '1.3.6.1.4.1.1466.115.121.1.4': (format_binary, None), # Audio [OBSOLETE] + '1.3.6.1.4.1.1466.115.121.1.5': (format_binary, None), # Binary [OBSOLETE] + '1.3.6.1.4.1.1466.115.121.1.6': (format_unicode, None), # Bit String + '1.3.6.1.4.1.1466.115.121.1.7': (format_boolean, validate_boolean), # Boolean + '1.3.6.1.4.1.1466.115.121.1.8': (format_binary, None), # Certificate [OBSOLETE] + '1.3.6.1.4.1.1466.115.121.1.9': (format_binary, None), # Certificate List [OBSOLETE] + '1.3.6.1.4.1.1466.115.121.1.10': (format_binary, None), # Certificate Pair [OBSOLETE] + '1.3.6.1.4.1.1466.115.121.1.11': (format_unicode, None), # Country String + '1.3.6.1.4.1.1466.115.121.1.12': (format_unicode, None), # Distinguished name (DN) + '1.3.6.1.4.1.1466.115.121.1.13': (format_binary, None), # Data Quality Syntax [OBSOLETE] + '1.3.6.1.4.1.1466.115.121.1.14': (format_unicode, None), # Delivery method + '1.3.6.1.4.1.1466.115.121.1.15': (format_unicode, None), # Directory string + '1.3.6.1.4.1.1466.115.121.1.16': (format_unicode, None), # DIT Content Rule Description + '1.3.6.1.4.1.1466.115.121.1.17': (format_unicode, None), # DIT Structure Rule Description + '1.3.6.1.4.1.1466.115.121.1.18': (format_binary, None), # DL Submit Permission [OBSOLETE] + '1.3.6.1.4.1.1466.115.121.1.19': (format_binary, None), # DSA Quality Syntax [OBSOLETE] + '1.3.6.1.4.1.1466.115.121.1.20': (format_binary, None), # DSE Type [OBSOLETE] + '1.3.6.1.4.1.1466.115.121.1.21': (format_binary, None), # Enhanced Guide + '1.3.6.1.4.1.1466.115.121.1.22': (format_unicode, None), # Facsimile Telephone Number + '1.3.6.1.4.1.1466.115.121.1.23': (format_binary, None), # Fax + '1.3.6.1.4.1.1466.115.121.1.24': (format_time, validate_time), # Generalized time + '1.3.6.1.4.1.1466.115.121.1.25': (format_binary, None), # Guide [OBSOLETE] + '1.3.6.1.4.1.1466.115.121.1.26': (format_unicode, None), # IA5 string + '1.3.6.1.4.1.1466.115.121.1.27': (format_integer, validate_integer), # Integer + '1.3.6.1.4.1.1466.115.121.1.28': (format_binary, None), # JPEG + '1.3.6.1.4.1.1466.115.121.1.29': (format_binary, None), # Master and Shadow Access Points [OBSOLETE] + '1.3.6.1.4.1.1466.115.121.1.30': (format_unicode, None), # Matching rule description + '1.3.6.1.4.1.1466.115.121.1.31': (format_unicode, None), # Matching rule use description + '1.3.6.1.4.1.1466.115.121.1.32': (format_unicode, None), # Mail Preference [OBSOLETE] + '1.3.6.1.4.1.1466.115.121.1.33': (format_unicode, None), # MHS OR Address [OBSOLETE] + '1.3.6.1.4.1.1466.115.121.1.34': (format_unicode, None), # Name and optional UID + '1.3.6.1.4.1.1466.115.121.1.35': (format_unicode, None), # Name form description + '1.3.6.1.4.1.1466.115.121.1.36': (format_unicode, None), # Numeric string + '1.3.6.1.4.1.1466.115.121.1.37': (format_unicode, None), # Object class description + '1.3.6.1.4.1.1466.115.121.1.38': (format_unicode, None), # OID + '1.3.6.1.4.1.1466.115.121.1.39': (format_unicode, None), # Other mailbox + '1.3.6.1.4.1.1466.115.121.1.40': (format_binary, None), # Octet string + '1.3.6.1.4.1.1466.115.121.1.41': (format_unicode, None), # Postal address + '1.3.6.1.4.1.1466.115.121.1.42': (format_binary, None), # Protocol Information [OBSOLETE] + '1.3.6.1.4.1.1466.115.121.1.43': (format_binary, None), # Presentation Address [OBSOLETE] + '1.3.6.1.4.1.1466.115.121.1.44': (format_unicode, None), # Printable string + '1.3.6.1.4.1.1466.115.121.1.45': (format_binary, None), # Subtree specification [OBSOLETE + '1.3.6.1.4.1.1466.115.121.1.46': (format_binary, None), # Supplier Information [OBSOLETE] + '1.3.6.1.4.1.1466.115.121.1.47': (format_binary, None), # Supplier Or Consumer [OBSOLETE] + '1.3.6.1.4.1.1466.115.121.1.48': (format_binary, None), # Supplier And Consumer [OBSOLETE] + '1.3.6.1.4.1.1466.115.121.1.49': (format_binary, None), # Supported Algorithm [OBSOLETE] + '1.3.6.1.4.1.1466.115.121.1.50': (format_unicode, None), # Telephone number + '1.3.6.1.4.1.1466.115.121.1.51': (format_unicode, None), # Teletex terminal identifier + '1.3.6.1.4.1.1466.115.121.1.52': (format_unicode, None), # Teletex number + '1.3.6.1.4.1.1466.115.121.1.53': (format_time, validate_time), # Utc time (deprecated) + '1.3.6.1.4.1.1466.115.121.1.54': (format_unicode, None), # LDAP syntax description + '1.3.6.1.4.1.1466.115.121.1.55': (format_binary, None), # Modify rights [OBSOLETE] + '1.3.6.1.4.1.1466.115.121.1.56': (format_binary, None), # LDAP Schema Definition [OBSOLETE] + '1.3.6.1.4.1.1466.115.121.1.57': (format_unicode, None), # LDAP Schema Description [OBSOLETE] + '1.3.6.1.4.1.1466.115.121.1.58': (format_unicode, None), # Substring assertion + '1.3.6.1.1.16.1': (format_uuid, validate_uuid), # UUID + '1.3.6.1.1.16.4': (format_uuid, validate_uuid), # entryUUID (RFC 4530) + '2.16.840.1.113719.1.1.4.1.501': (format_uuid, validate_guid), # GUID (Novell) + '2.16.840.1.113719.1.1.5.1.0': (format_binary, None), # Unknown (Novell) + '2.16.840.1.113719.1.1.5.1.6': (format_unicode, None), # Case Ignore List (Novell) + '2.16.840.1.113719.1.1.5.1.12': (format_binary, None), # Tagged Data (Novell) + '2.16.840.1.113719.1.1.5.1.13': (format_binary, None), # Octet List (Novell) + '2.16.840.1.113719.1.1.5.1.14': (format_unicode, None), # Tagged String (Novell) + '2.16.840.1.113719.1.1.5.1.15': (format_unicode, None), # Tagged Name And String (Novell) + '2.16.840.1.113719.1.1.5.1.16': (format_binary, None), # NDS Replica Pointer (Novell) + '2.16.840.1.113719.1.1.5.1.17': (format_unicode, None), # NDS ACL (Novell) + '2.16.840.1.113719.1.1.5.1.19': (format_time, validate_time), # NDS Timestamp (Novell) + '2.16.840.1.113719.1.1.5.1.22': (format_integer, validate_integer), # Counter (Novell) + '2.16.840.1.113719.1.1.5.1.23': (format_unicode, None), # Tagged Name (Novell) + '2.16.840.1.113719.1.1.5.1.25': (format_unicode, None), # Typed Name (Novell) + 'supportedldapversion': (format_integer, None), # supportedLdapVersion (Microsoft) + 'octetstring': (format_binary, validate_uuid_le), # octect string (Microsoft) + '1.2.840.113556.1.4.2': (format_uuid_le, validate_uuid_le), # object guid (Microsoft) + '1.2.840.113556.1.4.13': (format_ad_timestamp, validate_ad_timestamp), # builtinCreationTime (Microsoft) + '1.2.840.113556.1.4.26': (format_ad_timestamp, validate_ad_timestamp), # creationTime (Microsoft) + '1.2.840.113556.1.4.49': (format_ad_timestamp, validate_ad_timestamp), # badPasswordTime (Microsoft) + '1.2.840.113556.1.4.51': (format_ad_timestamp, validate_ad_timestamp), # lastLogoff (Microsoft) + '1.2.840.113556.1.4.52': (format_ad_timestamp, validate_ad_timestamp), # lastLogon (Microsoft) + '1.2.840.113556.1.4.60': (format_ad_timedelta, validate_ad_timedelta), # lockoutDuration (Microsoft) + '1.2.840.113556.1.4.61': (format_ad_timedelta, validate_ad_timedelta), # lockOutObservationWindow (Microsoft) + '1.2.840.113556.1.4.74': (format_ad_timedelta, validate_ad_timedelta), # maxPwdAge (Microsoft) + '1.2.840.113556.1.4.78': (format_ad_timedelta, validate_ad_timedelta), # minPwdAge (Microsoft) + '1.2.840.113556.1.4.96': (format_ad_timestamp, validate_zero_and_minus_one_and_positive_int), # pwdLastSet (Microsoft, can be set to -1 only) + '1.2.840.113556.1.4.146': (format_sid, validate_sid), # objectSid (Microsoft) + '1.2.840.113556.1.4.159': (format_ad_timestamp, validate_ad_timestamp), # accountExpires (Microsoft) + '1.2.840.113556.1.4.662': (format_ad_timestamp, validate_ad_timestamp), # lockoutTime (Microsoft) + '1.2.840.113556.1.4.1696': (format_ad_timestamp, validate_ad_timestamp), # lastLogonTimestamp (Microsoft) + '1.3.6.1.4.1.42.2.27.8.1.17': (format_time_with_0_year, validate_time_with_0_year) # pwdAccountLockedTime (Novell) +} + + +def find_attribute_helpers(attr_type, name, custom_formatter): + """ + Tries to format following the OIDs info and format_helper specification. + Search for attribute oid, then attribute name (can be multiple), then attribute syntax + Precedence is: + 1. attribute name + 2. attribute oid(from schema) + 3. attribute names (from oid_info) + 4. attribute syntax (from schema) + Custom formatters can be defined in Server object and have precedence over the standard_formatters + If no formatter is found the raw_value is returned as bytes. + Attributes defined as SINGLE_VALUE in schema are returned as a single object, otherwise are returned as a list of object + Formatter functions can return any kind of object + return a tuple (formatter, validator) + """ + formatter = None + if custom_formatter and isinstance(custom_formatter, dict): # if custom formatters are defined they have precedence over the standard formatters + if name in custom_formatter: # search for attribute name, as returned by the search operation + formatter = custom_formatter[name] + + if not formatter and attr_type and attr_type.oid in custom_formatter: # search for attribute oid as returned by schema + formatter = custom_formatter[attr_type.oid] + if not formatter and attr_type and attr_type.oid_info: + if isinstance(attr_type.oid_info[2], SEQUENCE_TYPES): # search for multiple names defined in oid_info + for attr_name in attr_type.oid_info[2]: + if attr_name in custom_formatter: + formatter = custom_formatter[attr_name] + break + elif attr_type.oid_info[2] in custom_formatter: # search for name defined in oid_info + formatter = custom_formatter[attr_type.oid_info[2]] + + if not formatter and attr_type and attr_type.syntax in custom_formatter: # search for syntax defined in schema + formatter = custom_formatter[attr_type.syntax] + + if not formatter and name in standard_formatter: # search for attribute name, as returned by the search operation + formatter = standard_formatter[name] + + if not formatter and attr_type and attr_type.oid in standard_formatter: # search for attribute oid as returned by schema + formatter = standard_formatter[attr_type.oid] + + if not formatter and attr_type and attr_type.oid_info: + if isinstance(attr_type.oid_info[2], SEQUENCE_TYPES): # search for multiple names defined in oid_info + for attr_name in attr_type.oid_info[2]: + if attr_name in standard_formatter: + formatter = standard_formatter[attr_name] + break + elif attr_type.oid_info[2] in standard_formatter: # search for name defined in oid_info + formatter = standard_formatter[attr_type.oid_info[2]] + if not formatter and attr_type and attr_type.syntax in standard_formatter: # search for syntax defined in schema + formatter = standard_formatter[attr_type.syntax] + + if formatter is None: + return None, None + + return formatter + + +def format_attribute_values(schema, name, values, custom_formatter): + if not values: # RFCs states that attributes must always have values, but a flaky server returns empty values too + return [] + + if not isinstance(values, SEQUENCE_TYPES): + values = [values] + + if schema and schema.attribute_types and name in schema.attribute_types: + attr_type = schema.attribute_types[name] + else: + attr_type = None + + attribute_helpers = find_attribute_helpers(attr_type, name, custom_formatter) + if not isinstance(attribute_helpers, tuple): # custom formatter + formatter = attribute_helpers + else: + formatter = format_unicode if not attribute_helpers[0] else attribute_helpers[0] + + formatted_values = [formatter(raw_value) for raw_value in values] # executes formatter + if formatted_values: + return formatted_values[0] if (attr_type and attr_type.single_value) else formatted_values + else: # RFCs states that attributes must always have values, but AD return empty values in DirSync + return [] + + +def find_attribute_validator(schema, name, custom_validator): + if schema and schema.attribute_types and name in schema.attribute_types: + attr_type = schema.attribute_types[name] + else: + attr_type = None + + attribute_helpers = find_attribute_helpers(attr_type, name, custom_validator) + if not isinstance(attribute_helpers, tuple): # custom validator + validator = attribute_helpers + else: + if not attribute_helpers[1]: + if attr_type and attr_type.single_value: + validator = validate_generic_single_value # validate only single value + else: + validator = always_valid # unknown syntax, accepts single and multi value + else: + validator = attribute_helpers[1] + return validator diff --git a/server/www/packages/packages-linux/x64/ldap3/protocol/formatters/validators.py b/server/www/packages/packages-linux/x64/ldap3/protocol/formatters/validators.py index fff2198..d5511b7 100644 --- a/server/www/packages/packages-linux/x64/ldap3/protocol/formatters/validators.py +++ b/server/www/packages/packages-linux/x64/ldap3/protocol/formatters/validators.py @@ -1,461 +1,489 @@ -""" -""" - -# Created on 2016.08.09 -# -# Author: Giovanni Cannata -# -# Copyright 2016 - 2018 Giovanni Cannata -# -# This file is part of ldap3. -# -# ldap3 is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# ldap3 is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with ldap3 in the COPYING and COPYING.LESSER files. -# If not, see . -from binascii import a2b_hex -from datetime import datetime -from calendar import timegm -from uuid import UUID -from struct import pack - - -from ... import SEQUENCE_TYPES, STRING_TYPES, NUMERIC_TYPES, INTEGER_TYPES -from .formatters import format_time, format_ad_timestamp -from ...utils.conv import to_raw, to_unicode, ldap_escape_to_bytes - -# Validators return True if value is valid, False if value is not valid, -# or a value different from True and False that is a valid value to substitute to the input value - - -def check_type(input_value, value_type): - if isinstance(input_value, value_type): - return True - - if isinstance(input_value, SEQUENCE_TYPES): - for value in input_value: - if not isinstance(value, value_type): - return False - return True - - return False - - -# noinspection PyUnusedLocal -def always_valid(input_value): - return True - - -def validate_generic_single_value(input_value): - if not isinstance(input_value, SEQUENCE_TYPES): - return True - - try: # object couldn't have a __len__ method - if len(input_value) == 1: - return True - except Exception: - pass - - return False - - -def validate_zero_and_minus_one_and_positive_int(input_value): - """Accept -1 only (used by pwdLastSet in AD) - """ - if not isinstance(input_value, SEQUENCE_TYPES): - if isinstance(input_value, NUMERIC_TYPES) or isinstance(input_value, STRING_TYPES): - return True if int(input_value) >= -1 else False - return False - else: - if len(input_value) == 1 and (isinstance(input_value[0], NUMERIC_TYPES) or isinstance(input_value[0], STRING_TYPES)): - return True if int(input_value[0]) >= -1 else False - - return False - - -def validate_integer(input_value): - if check_type(input_value, (float, bool)): - return False - if check_type(input_value, INTEGER_TYPES): - return True - - if not isinstance(input_value, SEQUENCE_TYPES): - sequence = False - input_value = [input_value] - else: - sequence = True # indicates if a sequence must be returned - - valid_values = [] # builds a list of valid int values - from decimal import Decimal, InvalidOperation - for element in input_value: - try: # try to convert any type to int, an invalid conversion raise TypeError or ValueError, doublecheck with Decimal type, if both are valid and equal then then int() value is used - value = to_unicode(element) if isinstance(element, bytes) else element - decimal_value = Decimal(value) - int_value = int(value) - if decimal_value == int_value: - valid_values.append(int_value) - else: - return False - except (ValueError, TypeError, InvalidOperation): - return False - - if sequence: - return valid_values - else: - return valid_values[0] - - -def validate_bytes(input_value): - return check_type(input_value, bytes) - - -def validate_boolean(input_value): - # it could be a real bool or the string TRUE or FALSE, # only a single valued is allowed - if validate_generic_single_value(input_value): # valid only if a single value or a sequence with a single element - if isinstance(input_value, SEQUENCE_TYPES): - input_value = input_value[0] - if isinstance(input_value, bool): - if input_value: - return 'TRUE' - else: - return 'FALSE' - if str is not bytes and isinstance(input_value, bytes): # python3 try to converts bytes to string - input_value = to_unicode(input_value) - if isinstance(input_value, STRING_TYPES): - if input_value.lower() == 'true': - return 'TRUE' - elif input_value.lower() == 'false': - return 'FALSE' - return False - - -def validate_time_with_0_year(input_value): - # validates generalized time but accept a 0000 year too - # if datetime object doesn't have a timezone it's considered local time and is adjusted to UTC - if not isinstance(input_value, SEQUENCE_TYPES): - sequence = False - input_value = [input_value] - else: - sequence = True # indicates if a sequence must be returned - - valid_values = [] - changed = False - for element in input_value: - if str is not bytes and isinstance(element, bytes): # python3 try to converts bytes to string - element = to_unicode(element) - if isinstance(element, STRING_TYPES): # tries to check if it is already be a Generalized Time - if element.startswith('0000') or isinstance(format_time(to_raw(element)), datetime): # valid Generalized Time string - valid_values.append(element) - else: - return False - elif isinstance(element, datetime): - changed = True - if element.tzinfo: # a datetime with a timezone - valid_values.append(element.strftime('%Y%m%d%H%M%S%z')) - else: # datetime without timezone, assumed local and adjusted to UTC - offset = datetime.now() - datetime.utcnow() - valid_values.append((element - offset).strftime('%Y%m%d%H%M%SZ')) - else: - return False - - if changed: - if sequence: - return valid_values - else: - return valid_values[0] - else: - return True - - -def validate_time(input_value): - # if datetime object doesn't have a timezone it's considered local time and is adjusted to UTC - if not isinstance(input_value, SEQUENCE_TYPES): - sequence = False - input_value = [input_value] - else: - sequence = True # indicates if a sequence must be returned - - valid_values = [] - changed = False - for element in input_value: - if str is not bytes and isinstance(element, bytes): # python3 try to converts bytes to string - element = to_unicode(element) - if isinstance(element, STRING_TYPES): # tries to check if it is already be a Generalized Time - if isinstance(format_time(to_raw(element)), datetime): # valid Generalized Time string - valid_values.append(element) - else: - return False - elif isinstance(element, datetime): - changed = True - if element.tzinfo: # a datetime with a timezone - valid_values.append(element.strftime('%Y%m%d%H%M%S%z')) - else: # datetime without timezone, assumed local and adjusted to UTC - offset = datetime.now() - datetime.utcnow() - valid_values.append((element - offset).strftime('%Y%m%d%H%M%SZ')) - else: - return False - - if changed: - if sequence: - return valid_values - else: - return valid_values[0] - else: - return True - - -def validate_ad_timestamp(input_value): - """ - Active Directory stores date/time values as the number of 100-nanosecond intervals - that have elapsed since the 0 hour on January 1, 1601 till the date/time that is being stored. - The time is always stored in Greenwich Mean Time (GMT) in the Active Directory. - """ - if not isinstance(input_value, SEQUENCE_TYPES): - sequence = False - input_value = [input_value] - else: - sequence = True # indicates if a sequence must be returned - - valid_values = [] - changed = False - for element in input_value: - if str is not bytes and isinstance(element, bytes): # python3 try to converts bytes to string - element = to_unicode(element) - if isinstance(element, NUMERIC_TYPES): - if 0 <= element <= 9223372036854775807: # min and max for the AD timestamp starting from 12:00 AM January 1, 1601 - valid_values.append(element) - else: - return False - elif isinstance(element, STRING_TYPES): # tries to check if it is already be a AD timestamp - if isinstance(format_ad_timestamp(to_raw(element)), datetime): # valid Generalized Time string - valid_values.append(element) - else: - return False - elif isinstance(element, datetime): - changed = True - if element.tzinfo: # a datetime with a timezone - valid_values.append(to_raw((timegm(element.utctimetuple()) + 11644473600) * 10000000, encoding='ascii')) - else: # datetime without timezone, assumed local and adjusted to UTC - offset = datetime.now() - datetime.utcnow() - valid_values.append(to_raw((timegm((element - offset).timetuple()) + 11644473600) * 10000000, encoding='ascii')) - else: - return False - - if changed: - if sequence: - return valid_values - else: - return valid_values[0] - else: - return True - - -def validate_guid(input_value): - """ - object guid in uuid format (Novell eDirectory) - """ - if not isinstance(input_value, SEQUENCE_TYPES): - sequence = False - input_value = [input_value] - else: - sequence = True # indicates if a sequence must be returned - - valid_values = [] - changed = False - for element in input_value: - if isinstance(element, STRING_TYPES): - try: - valid_values.append(UUID(element).bytes) - changed = True - except ValueError: # try if the value is an escaped byte sequence - try: - valid_values.append(UUID(element.replace('\\', '')).bytes) - changed = True - continue - except ValueError: - if str is not bytes: # python 3 - pass - else: - valid_values.append(element) - continue - return False - elif isinstance(element, (bytes, bytearray)): # assumes bytes are valid - valid_values.append(element) - else: - return False - - if changed: - if sequence: - return valid_values - else: - return valid_values[0] - else: - return True - -def validate_uuid(input_value): - """ - object entryUUID in uuid format - """ - if not isinstance(input_value, SEQUENCE_TYPES): - sequence = False - input_value = [input_value] - else: - sequence = True # indicates if a sequence must be returned - - valid_values = [] - changed = False - for element in input_value: - if isinstance(element, STRING_TYPES): - try: - valid_values.append(str(UUID(element))) - changed = True - except ValueError: # try if the value is an escaped byte sequence - try: - valid_values.append(str(UUID(element.replace('\\', '')))) - changed = True - continue - except ValueError: - if str is not bytes: # python 3 - pass - else: - valid_values.append(element) - continue - return False - elif isinstance(element, (bytes, bytearray)): # assumes bytes are valid - valid_values.append(element) - else: - return False - - if changed: - if sequence: - return valid_values - else: - return valid_values[0] - else: - return True - - -def validate_uuid_le(input_value): - """ - Active Directory stores objectGUID in uuid_le format, follows RFC4122 and MS-DTYP: - "{07039e68-4373-264d-a0a7-07039e684373}": string representation big endian, converted to little endian (with or without brace curles) - "689e030773434d26a7a007039e684373": packet representation, already in little endian - "\68\9e\03\07\73\43\4d\26\a7\a0\07\03\9e\68\43\73": bytes representation, already in little endian - byte sequence: already in little endian - - """ - if not isinstance(input_value, SEQUENCE_TYPES): - sequence = False - input_value = [input_value] - else: - sequence = True # indicates if a sequence must be returned - - valid_values = [] - changed = False - for element in input_value: - if isinstance(element, STRING_TYPES): - if element[0] == '{' and element[-1] == '}': - valid_values.append(UUID(hex=element).bytes_le) # string representation, value in big endian, converts to little endian - changed = True - elif '-' in element: - valid_values.append(UUID(hex=element).bytes_le) # string representation, value in big endian, converts to little endian - changed = True - elif '\\' in element: - valid_values.append(UUID(bytes_le=ldap_escape_to_bytes(element)).bytes_le) # byte representation, value in little endian - changed = True - elif '-' not in element: # value in little endian - valid_values.append(UUID(bytes_le=a2b_hex(element)).bytes_le) # packet representation, value in little endian, converts to little endian - changed = True - elif isinstance(element, (bytes, bytearray)): # assumes bytes are valid uuid - valid_values.append(element) # value is untouched, must be in little endian - else: - return False - - if changed: - if sequence: - return valid_values - else: - return valid_values[0] - else: - return True - - -def validate_sid(input_value): - """ - SID= "S-1-" IdentifierAuthority 1*SubAuthority - IdentifierAuthority= IdentifierAuthorityDec / IdentifierAuthorityHex - ; If the identifier authority is < 2^32, the - ; identifier authority is represented as a decimal - ; number - ; If the identifier authority is >= 2^32, - ; the identifier authority is represented in - ; hexadecimal - IdentifierAuthorityDec = 1*10DIGIT - ; IdentifierAuthorityDec, top level authority of a - ; security identifier is represented as a decimal number - IdentifierAuthorityHex = "0x" 12HEXDIG - ; IdentifierAuthorityHex, the top-level authority of a - ; security identifier is represented as a hexadecimal number - SubAuthority= "-" 1*10DIGIT - ; Sub-Authority is always represented as a decimal number - ; No leading "0" characters are allowed when IdentifierAuthority - ; or SubAuthority is represented as a decimal number - ; All hexadecimal digits must be output in string format, - ; pre-pended by "0x" - - Revision (1 byte): An 8-bit unsigned integer that specifies the revision level of the SID. This value MUST be set to 0x01. - SubAuthorityCount (1 byte): An 8-bit unsigned integer that specifies the number of elements in the SubAuthority array. The maximum number of elements allowed is 15. - IdentifierAuthority (6 bytes): A SID_IDENTIFIER_AUTHORITY structure that indicates the authority under which the SID was created. It describes the entity that created the SID. The Identifier Authority value {0,0,0,0,0,5} denotes SIDs created by the NT SID authority. - SubAuthority (variable): A variable length array of unsigned 32-bit integers that uniquely identifies a principal relative to the IdentifierAuthority. Its length is determined by SubAuthorityCount. - - If you have a SID like S-a-b-c-d-e-f-g-... - - Then the bytes are - a (revision) - N (number of dashes minus two) - bbbbbb (six bytes of "b" treated as a 48-bit number in big-endian format) - cccc (four bytes of "c" treated as a 32-bit number in little-endian format) - dddd (four bytes of "d" treated as a 32-bit number in little-endian format) - eeee (four bytes of "e" treated as a 32-bit number in little-endian format) - ffff (four bytes of "f" treated as a 32-bit number in little-endian format) - - """ - if not isinstance(input_value, SEQUENCE_TYPES): - sequence = False - input_value = [input_value] - else: - sequence = True # indicates if a sequence must be returned - - valid_values = [] - changed = False - for element in input_value: - if isinstance(element, STRING_TYPES): - if element.startswith('S-'): - parts = element.split('-') - sid_bytes = pack('q', int(parts[2]))[2:] # authority (in dec) - else: - sid_bytes += pack('>q', int(parts[2], 16))[2:] # authority (in hex) - for sub_auth in parts[3:]: - sid_bytes += pack('. +from binascii import a2b_hex +from datetime import datetime +from calendar import timegm +from uuid import UUID +from struct import pack + + +from ... import SEQUENCE_TYPES, STRING_TYPES, NUMERIC_TYPES, INTEGER_TYPES +from .formatters import format_time, format_ad_timestamp +from ...utils.conv import to_raw, to_unicode, ldap_escape_to_bytes, escape_bytes + +# Validators return True if value is valid, False if value is not valid, +# or a value different from True and False that is a valid value to substitute to the input value + + +def check_type(input_value, value_type): + if isinstance(input_value, value_type): + return True + + if isinstance(input_value, SEQUENCE_TYPES): + for value in input_value: + if not isinstance(value, value_type): + return False + return True + + return False + + +# noinspection PyUnusedLocal +def always_valid(input_value): + return True + + +def validate_generic_single_value(input_value): + if not isinstance(input_value, SEQUENCE_TYPES): + return True + + try: # object couldn't have a __len__ method + if len(input_value) == 1: + return True + except Exception: + pass + + return False + + +def validate_zero_and_minus_one_and_positive_int(input_value): + """Accept -1 and 0 only (used by pwdLastSet in AD) + """ + if not isinstance(input_value, SEQUENCE_TYPES): + if isinstance(input_value, NUMERIC_TYPES) or isinstance(input_value, STRING_TYPES): + return True if int(input_value) >= -1 else False + return False + else: + if len(input_value) == 1 and (isinstance(input_value[0], NUMERIC_TYPES) or isinstance(input_value[0], STRING_TYPES)): + return True if int(input_value[0]) >= -1 else False + + return False + + +def validate_integer(input_value): + if check_type(input_value, (float, bool)): + return False + if check_type(input_value, INTEGER_TYPES): + return True + + if not isinstance(input_value, SEQUENCE_TYPES): + sequence = False + input_value = [input_value] + else: + sequence = True # indicates if a sequence must be returned + + valid_values = [] # builds a list of valid int values + from decimal import Decimal, InvalidOperation + for element in input_value: + try: #try to convert any type to int, an invalid conversion raise TypeError or ValueError, doublecheck with Decimal type, if both are valid and equal then then int() value is used + value = to_unicode(element) if isinstance(element, bytes) else element + decimal_value = Decimal(value) + int_value = int(value) + if decimal_value == int_value: + valid_values.append(int_value) + else: + return False + except (ValueError, TypeError, InvalidOperation): + return False + + if sequence: + return valid_values + else: + return valid_values[0] + + +def validate_bytes(input_value): + return check_type(input_value, bytes) + + +def validate_boolean(input_value): + # it could be a real bool or the string TRUE or FALSE, # only a single valued is allowed + if validate_generic_single_value(input_value): # valid only if a single value or a sequence with a single element + if isinstance(input_value, SEQUENCE_TYPES): + input_value = input_value[0] + if isinstance(input_value, bool): + if input_value: + return 'TRUE' + else: + return 'FALSE' + if str is not bytes and isinstance(input_value, bytes): # python3 try to converts bytes to string + input_value = to_unicode(input_value) + if isinstance(input_value, STRING_TYPES): + if input_value.lower() == 'true': + return 'TRUE' + elif input_value.lower() == 'false': + return 'FALSE' + return False + + +def validate_time_with_0_year(input_value): + # validates generalized time but accept a 0000 year too + # if datetime object doesn't have a timezone it's considered local time and is adjusted to UTC + if not isinstance(input_value, SEQUENCE_TYPES): + sequence = False + input_value = [input_value] + else: + sequence = True # indicates if a sequence must be returned + + valid_values = [] + changed = False + for element in input_value: + if str is not bytes and isinstance(element, bytes): # python3 try to converts bytes to string + element = to_unicode(element) + if isinstance(element, STRING_TYPES): # tries to check if it is already be a Generalized Time + if element.startswith('0000') or isinstance(format_time(to_raw(element)), datetime): # valid Generalized Time string + valid_values.append(element) + else: + return False + elif isinstance(element, datetime): + changed = True + if element.tzinfo: # a datetime with a timezone + valid_values.append(element.strftime('%Y%m%d%H%M%S%z')) + else: # datetime without timezone, assumed local and adjusted to UTC + offset = datetime.now() - datetime.utcnow() + valid_values.append((element - offset).strftime('%Y%m%d%H%M%SZ')) + else: + return False + + if changed: + if sequence: + return valid_values + else: + return valid_values[0] + else: + return True + + +def validate_time(input_value): + # if datetime object doesn't have a timezone it's considered local time and is adjusted to UTC + if not isinstance(input_value, SEQUENCE_TYPES): + sequence = False + input_value = [input_value] + else: + sequence = True # indicates if a sequence must be returned + + valid_values = [] + changed = False + for element in input_value: + if str is not bytes and isinstance(element, bytes): # python3 try to converts bytes to string + element = to_unicode(element) + if isinstance(element, STRING_TYPES): # tries to check if it is already be a Generalized Time + if isinstance(format_time(to_raw(element)), datetime): # valid Generalized Time string + valid_values.append(element) + else: + return False + elif isinstance(element, datetime): + changed = True + if element.tzinfo: # a datetime with a timezone + valid_values.append(element.strftime('%Y%m%d%H%M%S%z')) + else: # datetime without timezone, assumed local and adjusted to UTC + offset = datetime.now() - datetime.utcnow() + valid_values.append((element - offset).strftime('%Y%m%d%H%M%SZ')) + else: + return False + + if changed: + if sequence: + return valid_values + else: + return valid_values[0] + else: + return True + + +def validate_ad_timestamp(input_value): + """ + Active Directory stores date/time values as the number of 100-nanosecond intervals + that have elapsed since the 0 hour on January 1, 1601 till the date/time that is being stored. + The time is always stored in Greenwich Mean Time (GMT) in the Active Directory. + """ + if not isinstance(input_value, SEQUENCE_TYPES): + sequence = False + input_value = [input_value] + else: + sequence = True # indicates if a sequence must be returned + + valid_values = [] + changed = False + for element in input_value: + if str is not bytes and isinstance(element, bytes): # python3 try to converts bytes to string + element = to_unicode(element) + if isinstance(element, NUMERIC_TYPES): + if 0 <= element <= 9223372036854775807: # min and max for the AD timestamp starting from 12:00 AM January 1, 1601 + valid_values.append(element) + else: + return False + elif isinstance(element, STRING_TYPES): # tries to check if it is already be a AD timestamp + if isinstance(format_ad_timestamp(to_raw(element)), datetime): # valid Generalized Time string + valid_values.append(element) + else: + return False + elif isinstance(element, datetime): + changed = True + if element.tzinfo: # a datetime with a timezone + valid_values.append(to_raw((timegm(element.utctimetuple()) + 11644473600) * 10000000, encoding='ascii')) + else: # datetime without timezone, assumed local and adjusted to UTC + offset = datetime.now() - datetime.utcnow() + valid_values.append(to_raw((timegm((element - offset).timetuple()) + 11644473600) * 10000000, encoding='ascii')) + else: + return False + + if changed: + if sequence: + return valid_values + else: + return valid_values[0] + else: + return True + + +def validate_ad_timedelta(input_value): + """ + Should be validated like an AD timestamp except that since it is a time + delta, it is stored as a negative number. + """ + if not isinstance(input_value, INTEGER_TYPES) or input_value > 0: + return False + return validate_ad_timestamp(input_value * -1) + + +def validate_guid(input_value): + """ + object guid in uuid format (Novell eDirectory) + """ + if not isinstance(input_value, SEQUENCE_TYPES): + sequence = False + input_value = [input_value] + else: + sequence = True # indicates if a sequence must be returned + + valid_values = [] + changed = False + for element in input_value: + if isinstance(element, STRING_TYPES): + try: + valid_values.append(UUID(element).bytes) + changed = True + except ValueError: # try if the value is an escaped byte sequence + try: + valid_values.append(UUID(element.replace('\\', '')).bytes) + changed = True + continue + except ValueError: + if str is not bytes: # python 3 + pass + else: + valid_values.append(element) + continue + return False + elif isinstance(element, (bytes, bytearray)): # assumes bytes are valid + valid_values.append(element) + else: + return False + + if changed: + if sequence: + return valid_values + else: + return valid_values[0] + else: + return True + + +def validate_uuid(input_value): + """ + object entryUUID in uuid format + """ + if not isinstance(input_value, SEQUENCE_TYPES): + sequence = False + input_value = [input_value] + else: + sequence = True # indicates if a sequence must be returned + + valid_values = [] + changed = False + for element in input_value: + if isinstance(element, STRING_TYPES): + try: + valid_values.append(str(UUID(element))) + changed = True + except ValueError: # try if the value is an escaped byte sequence + try: + valid_values.append(str(UUID(element.replace('\\', '')))) + changed = True + continue + except ValueError: + if str is not bytes: # python 3 + pass + else: + valid_values.append(element) + continue + return False + elif isinstance(element, (bytes, bytearray)): # assumes bytes are valid + valid_values.append(element) + else: + return False + + if changed: + if sequence: + return valid_values + else: + return valid_values[0] + else: + return True + + +def validate_uuid_le(input_value): + """ + Active Directory stores objectGUID in uuid_le format, follows RFC4122 and MS-DTYP: + "{07039e68-4373-264d-a0a7-07039e684373}": string representation big endian, converted to little endian (with or without brace curles) + "689e030773434d26a7a007039e684373": packet representation, already in little endian + "\68\9e\03\07\73\43\4d\26\a7\a0\07\03\9e\68\43\73": bytes representation, already in little endian + byte sequence: already in little endian + + """ + if not isinstance(input_value, SEQUENCE_TYPES): + sequence = False + input_value = [input_value] + else: + sequence = True # indicates if a sequence must be returned + + valid_values = [] + changed = False + for element in input_value: + error = False + if isinstance(element, STRING_TYPES): + if element[0] == '{' and element[-1] == '}': + try: + valid_values.append(UUID(hex=element).bytes_le) # string representation, value in big endian, converts to little endian + changed = True + except ValueError: + error = True + elif '-' in element: + try: + valid_values.append(UUID(hex=element).bytes_le) # string representation, value in big endian, converts to little endian + changed = True + except ValueError: + error = True + elif '\\' in element: + try: + uuid = UUID(bytes_le=ldap_escape_to_bytes(element)).bytes_le + uuid = escape_bytes(uuid) + valid_values.append(uuid) # byte representation, value in little endian + changed = True + except ValueError: + error = True + elif '-' not in element: # value in little endian + try: + valid_values.append(UUID(bytes_le=a2b_hex(element)).bytes_le) # packet representation, value in little endian, converts to little endian + changed = True + except ValueError: + error = True + if error and str == bytes: # python2 only assume value is bytes and valid + valid_values.append(element) # value is untouched, must be in little endian + elif isinstance(element, (bytes, bytearray)): # assumes bytes are valid uuid + valid_values.append(element) # value is untouched, must be in little endian + else: + return False + + if changed: + if sequence: + return valid_values + else: + return valid_values[0] + else: + return True + + +def validate_sid(input_value): + """ + SID= "S-1-" IdentifierAuthority 1*SubAuthority + IdentifierAuthority= IdentifierAuthorityDec / IdentifierAuthorityHex + ; If the identifier authority is < 2^32, the + ; identifier authority is represented as a decimal + ; number + ; If the identifier authority is >= 2^32, + ; the identifier authority is represented in + ; hexadecimal + IdentifierAuthorityDec = 1*10DIGIT + ; IdentifierAuthorityDec, top level authority of a + ; security identifier is represented as a decimal number + IdentifierAuthorityHex = "0x" 12HEXDIG + ; IdentifierAuthorityHex, the top-level authority of a + ; security identifier is represented as a hexadecimal number + SubAuthority= "-" 1*10DIGIT + ; Sub-Authority is always represented as a decimal number + ; No leading "0" characters are allowed when IdentifierAuthority + ; or SubAuthority is represented as a decimal number + ; All hexadecimal digits must be output in string format, + ; pre-pended by "0x" + + Revision (1 byte): An 8-bit unsigned integer that specifies the revision level of the SID. This value MUST be set to 0x01. + SubAuthorityCount (1 byte): An 8-bit unsigned integer that specifies the number of elements in the SubAuthority array. The maximum number of elements allowed is 15. + IdentifierAuthority (6 bytes): A SID_IDENTIFIER_AUTHORITY structure that indicates the authority under which the SID was created. It describes the entity that created the SID. The Identifier Authority value {0,0,0,0,0,5} denotes SIDs created by the NT SID authority. + SubAuthority (variable): A variable length array of unsigned 32-bit integers that uniquely identifies a principal relative to the IdentifierAuthority. Its length is determined by SubAuthorityCount. + + If you have a SID like S-a-b-c-d-e-f-g-... + + Then the bytes are + a (revision) + N (number of dashes minus two) + bbbbbb (six bytes of "b" treated as a 48-bit number in big-endian format) + cccc (four bytes of "c" treated as a 32-bit number in little-endian format) + dddd (four bytes of "d" treated as a 32-bit number in little-endian format) + eeee (four bytes of "e" treated as a 32-bit number in little-endian format) + ffff (four bytes of "f" treated as a 32-bit number in little-endian format) + + """ + if not isinstance(input_value, SEQUENCE_TYPES): + sequence = False + input_value = [input_value] + else: + sequence = True # indicates if a sequence must be returned + + valid_values = [] + changed = False + for element in input_value: + if isinstance(element, STRING_TYPES): + if element.startswith('S-'): + parts = element.split('-') + sid_bytes = pack('q', int(parts[2]))[2:] # authority (in dec) + else: + sid_bytes += pack('>q', int(parts[2], 16))[2:] # authority (in hex) + for sub_auth in parts[3:]: + sid_bytes += pack('= 1 and connection.sasl_credentials[0]: if connection.sasl_credentials[0] is True: @@ -70,9 +75,15 @@ def sasl_gssapi(connection, controls): target_name = gssapi.Name('ldap@' + connection.sasl_credentials[0], gssapi.NameType.hostbased_service) if len(connection.sasl_credentials) >= 2 and connection.sasl_credentials[1]: authz_id = connection.sasl_credentials[1].encode("utf-8") + if len(connection.sasl_credentials) >= 3 and connection.sasl_credentials[2]: + raw_creds = connection.sasl_credentials[2] if target_name is None: target_name = gssapi.Name('ldap@' + connection.server.host, gssapi.NameType.hostbased_service) - creds = gssapi.Credentials(name=gssapi.Name(connection.user), usage='initiate') if connection.user else None + + if raw_creds is not None: + creds = gssapi.Credentials(base=raw_creds, usage='initiate', store=connection.cred_store) + else: + creds = gssapi.Credentials(name=gssapi.Name(connection.user), usage='initiate', store=connection.cred_store) if connection.user else None ctx = gssapi.SecurityContext(name=target_name, mech=gssapi.MechType.kerberos, creds=creds) in_token = None try: diff --git a/server/www/packages/packages-linux/x64/ldap3/protocol/sasl/plain.py b/server/www/packages/packages-linux/x64/ldap3/protocol/sasl/plain.py index 1de2a36..5327275 100644 --- a/server/www/packages/packages-linux/x64/ldap3/protocol/sasl/plain.py +++ b/server/www/packages/packages-linux/x64/ldap3/protocol/sasl/plain.py @@ -5,7 +5,7 @@ # # Author: Giovanni Cannata # -# Copyright 2014 - 2018 Giovanni Cannata +# Copyright 2014 - 2019 Giovanni Cannata # # This file is part of ldap3. # diff --git a/server/www/packages/packages-linux/x64/ldap3/protocol/schemas/ad2012R2.py b/server/www/packages/packages-linux/x64/ldap3/protocol/schemas/ad2012R2.py index f583973..d0f3363 100644 --- a/server/www/packages/packages-linux/x64/ldap3/protocol/schemas/ad2012R2.py +++ b/server/www/packages/packages-linux/x64/ldap3/protocol/schemas/ad2012R2.py @@ -5,7 +5,7 @@ # # Author: Giovanni Cannata # -# Copyright 2014 - 2018 Giovanni Cannata +# Copyright 2014 - 2019 Giovanni Cannata # # This file is part of ldap3. # diff --git a/server/www/packages/packages-linux/x64/ldap3/protocol/schemas/ds389.py b/server/www/packages/packages-linux/x64/ldap3/protocol/schemas/ds389.py index 0ede92f..4d49331 100644 --- a/server/www/packages/packages-linux/x64/ldap3/protocol/schemas/ds389.py +++ b/server/www/packages/packages-linux/x64/ldap3/protocol/schemas/ds389.py @@ -5,7 +5,7 @@ # # Author: Giovanni Cannata # -# Copyright 2014 - 2018 Giovanni Cannata +# Copyright 2014 - 2019 Giovanni Cannata # # This file is part of ldap3. # diff --git a/server/www/packages/packages-linux/x64/ldap3/protocol/schemas/edir888.py b/server/www/packages/packages-linux/x64/ldap3/protocol/schemas/edir888.py index 630d7dc..80499a4 100644 --- a/server/www/packages/packages-linux/x64/ldap3/protocol/schemas/edir888.py +++ b/server/www/packages/packages-linux/x64/ldap3/protocol/schemas/edir888.py @@ -5,7 +5,7 @@ # # Author: Giovanni Cannata # -# Copyright 2014 - 2018 Giovanni Cannata +# Copyright 2014 - 2019 Giovanni Cannata # # This file is part of ldap3. # @@ -939,12 +939,7 @@ edir_8_8_8_dsa_info = """ "addEntryOps": [ "947" ], - "altServer": [ - "ldap://192.168.137.102:389/", - "ldaps://192.168.137.102:636/", - "ldap://192.168.137.103:389/", - "ldaps://192.168.137.103:636/" - ], + "altServer": [], "bindSecurityErrors": [ "3" ], diff --git a/server/www/packages/packages-linux/x64/ldap3/protocol/schemas/edir914.py b/server/www/packages/packages-linux/x64/ldap3/protocol/schemas/edir914.py new file mode 100644 index 0000000..f86c417 --- /dev/null +++ b/server/www/packages/packages-linux/x64/ldap3/protocol/schemas/edir914.py @@ -0,0 +1,1157 @@ +""" +""" + +# Created on 2019.08.31 +# +# Author: Giovanni Cannata +# +# Copyright 2014 - 2019 Giovanni Cannata +# +# This file is part of ldap3. +# +# ldap3 is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ldap3 is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with ldap3 in the COPYING and COPYING.LESSER files. +# If not, see . + +edir_9_1_4_schema = """ +{ + "raw": { + "attributeTypes": [ + "( 2.5.4.35 NAME 'userPassword' DESC 'Internal NDS policy forces this to be single-valued' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{128} USAGE directoryOperation )", + "( 2.5.18.1 NAME 'createTimestamp' DESC 'Operational Attribute' SYNTAX 1.3.6.1.4.1.1466.115.121.1.24 SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation )", + "( 2.5.18.2 NAME 'modifyTimestamp' DESC 'Operational Attribute' SYNTAX 1.3.6.1.4.1.1466.115.121.1.24 SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation )", + "( 2.5.18.10 NAME 'subschemaSubentry' DESC 'Operational Attribute' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 USAGE directoryOperation )", + "( 2.5.21.9 NAME 'structuralObjectClass' DESC 'Operational Attribute' SYNTAX 1.3.6.1.4.1.1466.115.121.1.38 SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation )", + "( 2.16.840.1.113719.1.27.4.49 NAME 'subordinateCount' DESC 'Operational Attribute' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation )", + "( 2.16.840.1.113719.1.27.4.48 NAME 'entryFlags' DESC 'Operational Attribute' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation )", + "( 2.16.840.1.113719.1.27.4.51 NAME 'federationBoundary' DESC 'Operational Attribute' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 NO-USER-MODIFICATION USAGE directoryOperation )", + "( 2.5.21.5 NAME 'attributeTypes' DESC 'Operational Attribute' SYNTAX 1.3.6.1.4.1.1466.115.121.1.3 USAGE directoryOperation )", + "( 2.5.21.6 NAME 'objectClasses' DESC 'Operational Attribute' SYNTAX 1.3.6.1.4.1.1466.115.121.1.37 USAGE directoryOperation )", + "( 1.3.6.1.1.20 NAME 'entryDN' DESC 'Operational Attribute' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation )", + "( 2.16.840.1.113719.1.1.4.1.2 NAME 'ACL' SYNTAX 2.16.840.1.113719.1.1.5.1.17 X-NDS_NONREMOVABLE '1' X-NDS_FILTERED_REQUIRED '1' )", + "( 2.5.4.1 NAME 'aliasedObjectName' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_NAME 'Aliased Object Name' X-NDS_NONREMOVABLE '1' X-NDS_FILTERED_REQUIRED '1' )", + "( 2.16.840.1.113719.1.1.4.1.6 NAME 'backLink' SYNTAX 2.16.840.1.113719.1.1.5.1.23 NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NAME 'Back Link' X-NDS_SERVER_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' X-NDS_FILTERED_REQUIRED '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.8 NAME 'binderyProperty' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NAME 'Bindery Property' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.7 NAME 'binderyObjectRestriction' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NAME 'Bindery Object Restriction' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.9 NAME 'binderyType' SYNTAX 1.3.6.1.4.1.1466.115.121.1.36{64512} SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NAME 'Bindery Type' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.11 NAME 'cAPrivateKey' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NAME 'CA Private Key' X-NDS_NONREMOVABLE '1' X-NDS_HIDDEN '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.12 NAME 'cAPublicKey' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NAME 'CA Public Key' X-NDS_PUBLIC_READ '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.10 NAME 'Cartridge' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} X-NDS_NONREMOVABLE '1' )", + "( 2.5.4.3 NAME ( 'cn' 'commonName' ) SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64} X-NDS_NAME 'CN' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '64' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.78 NAME 'printerConfiguration' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_NAME 'Printer Configuration' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.15 NAME 'Convergence' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27{1} SINGLE-VALUE X-NDS_UPPER_BOUND '1' X-NDS_NONREMOVABLE '1' )", + "( 2.5.4.6 NAME ( 'c' 'countryName' ) SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{2} SINGLE-VALUE X-NDS_NAME 'C' X-NDS_LOWER_BOUND '2' X-NDS_UPPER_BOUND '2' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.18 NAME 'defaultQueue' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_NAME 'Default Queue' X-NDS_SERVER_READ '1' X-NDS_NONREMOVABLE '1' )", + "( 2.5.4.13 NAME ( 'description' 'multiLineDescription' ) SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{1024} X-NDS_NAME 'Description' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '1024' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.64 NAME 'partitionCreationTime' SYNTAX 2.16.840.1.113719.1.1.5.1.19 SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NAME 'Partition Creation Time' X-NDS_PUBLIC_READ '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.5.4.23 NAME 'facsimileTelephoneNumber' SYNTAX 1.3.6.1.4.1.1466.115.121.1.22{64512} X-NDS_NAME 'Facsimile Telephone Number' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.117 NAME 'highConvergenceSyncInterval' SYNTAX 2.16.840.1.113719.1.1.5.1.27 SINGLE-VALUE X-NDS_NAME 'High Convergence Sync Interval' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.25 NAME 'groupMembership' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_NAME 'Group Membership' X-NDS_NAME_VALUE_ACCESS '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.26 NAME 'ndsHomeDirectory' SYNTAX 2.16.840.1.113719.1.1.5.1.15{255} SINGLE-VALUE X-NDS_NAME 'Home Directory' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '255' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.27 NAME 'hostDevice' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_NAME 'Host Device' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.28 NAME 'hostResourceName' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE X-NDS_NAME 'Host Resource Name' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.29 NAME 'hostServer' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_NAME 'Host Server' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.30 NAME 'inheritedACL' SYNTAX 2.16.840.1.113719.1.1.5.1.17 NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NAME 'Inherited ACL' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.5.4.7 NAME ( 'l' 'localityname' ) SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{128} X-NDS_NAME 'L' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '128' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.39 NAME 'loginAllowedTimeMap' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{42} SINGLE-VALUE X-NDS_NAME 'Login Allowed Time Map' X-NDS_LOWER_BOUND '42' X-NDS_UPPER_BOUND '42' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.40 NAME 'loginDisabled' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NAME 'Login Disabled' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.41 NAME 'loginExpirationTime' SYNTAX 1.3.6.1.4.1.1466.115.121.1.24 SINGLE-VALUE X-NDS_NAME 'Login Expiration Time' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.42 NAME 'loginGraceLimit' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NAME 'Login Grace Limit' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.43 NAME 'loginGraceRemaining' SYNTAX 2.16.840.1.113719.1.1.5.1.22 SINGLE-VALUE X-NDS_NAME 'Login Grace Remaining' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.44 NAME 'loginIntruderAddress' SYNTAX 2.16.840.1.113719.1.1.5.1.12 SINGLE-VALUE X-NDS_NAME 'Login Intruder Address' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.45 NAME 'loginIntruderAttempts' SYNTAX 2.16.840.1.113719.1.1.5.1.22 SINGLE-VALUE X-NDS_NAME 'Login Intruder Attempts' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.46 NAME 'loginIntruderLimit' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NAME 'Login Intruder Limit' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.31 NAME 'intruderAttemptResetInterval' SYNTAX 2.16.840.1.113719.1.1.5.1.27 SINGLE-VALUE X-NDS_NAME 'Intruder Attempt Reset Interval' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.47 NAME 'loginIntruderResetTime' SYNTAX 1.3.6.1.4.1.1466.115.121.1.24 SINGLE-VALUE X-NDS_NAME 'Login Intruder Reset Time' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.48 NAME 'loginMaximumSimultaneous' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NAME 'Login Maximum Simultaneous' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.49 NAME 'loginScript' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_NAME 'Login Script' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.50 NAME 'loginTime' SYNTAX 1.3.6.1.4.1.1466.115.121.1.24 SINGLE-VALUE X-NDS_NAME 'Login Time' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' )", + "( 2.5.4.31 NAME ( 'member' 'uniqueMember' ) SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_NAME 'Member' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.52 NAME 'Memory' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.22 NAME 'eMailAddress' SYNTAX 2.16.840.1.113719.1.1.5.1.14{64512} X-NDS_NAME 'EMail Address' X-NDS_PUBLIC_READ '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.55 NAME 'networkAddress' SYNTAX 2.16.840.1.113719.1.1.5.1.12 X-NDS_NAME 'Network Address' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.56 NAME 'networkAddressRestriction' SYNTAX 2.16.840.1.113719.1.1.5.1.12 X-NDS_NAME 'Network Address Restriction' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.57 NAME 'notify' SYNTAX 2.16.840.1.113719.1.1.5.1.25 X-NDS_NAME 'Notify' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.114 NAME 'Obituary' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NONREMOVABLE '1' X-NDS_FILTERED_REQUIRED '1' X-NDS_READ_FILTERED '1' )", + "( 2.5.4.0 NAME 'objectClass' SYNTAX 1.3.6.1.4.1.1466.115.121.1.38 X-NDS_NAME 'Object Class' X-NDS_PUBLIC_READ '1' X-NDS_NONREMOVABLE '1' X-NDS_FILTERED_REQUIRED '1' )", + "( 2.16.840.1.113719.1.1.4.1.59 NAME 'operator' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_NAME 'Operator' X-NDS_SERVER_READ '1' X-NDS_NONREMOVABLE '1' )", + "( 2.5.4.11 NAME ( 'ou' 'organizationalUnitName' ) SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64} X-NDS_NAME 'OU' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '64' X-NDS_NONREMOVABLE '1' )", + "( 2.5.4.10 NAME ( 'o' 'organizationname' ) SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64} X-NDS_NAME 'O' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '64' X-NDS_NONREMOVABLE '1' )", + "( 2.5.4.32 NAME 'owner' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_NAME 'Owner' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.63 NAME 'pageDescriptionLanguage' SYNTAX 1.3.6.1.4.1.1466.115.121.1.44{64} X-NDS_NAME 'Page Description Language' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '64' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.65 NAME 'passwordsUsed' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} USAGE directoryOperation X-NDS_NAME 'Passwords Used' X-NDS_NONREMOVABLE '1' X-NDS_HIDDEN '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.66 NAME 'passwordAllowChange' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NAME 'Password Allow Change' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.67 NAME 'passwordExpirationInterval' SYNTAX 2.16.840.1.113719.1.1.5.1.27 SINGLE-VALUE X-NDS_NAME 'Password Expiration Interval' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.68 NAME 'passwordExpirationTime' SYNTAX 1.3.6.1.4.1.1466.115.121.1.24 SINGLE-VALUE X-NDS_NAME 'Password Expiration Time' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.69 NAME 'passwordMinimumLength' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NAME 'Password Minimum Length' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.70 NAME 'passwordRequired' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NAME 'Password Required' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.71 NAME 'passwordUniqueRequired' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NAME 'Password Unique Required' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.72 NAME 'path' SYNTAX 2.16.840.1.113719.1.1.5.1.15{64512} X-NDS_NAME 'Path' X-NDS_NONREMOVABLE '1' )", + "( 2.5.4.19 NAME 'physicalDeliveryOfficeName' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{128} X-NDS_NAME 'Physical Delivery Office Name' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '128' X-NDS_NONREMOVABLE '1' )", + "( 2.5.4.16 NAME 'postalAddress' SYNTAX 1.3.6.1.4.1.1466.115.121.1.41{64512} X-NDS_NAME 'Postal Address' X-NDS_NONREMOVABLE '1' )", + "( 2.5.4.17 NAME 'postalCode' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{40} X-NDS_NAME 'Postal Code' X-NDS_UPPER_BOUND '40' X-NDS_NONREMOVABLE '1' )", + "( 2.5.4.18 NAME 'postOfficeBox' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{40} X-NDS_NAME 'Postal Office Box' X-NDS_UPPER_BOUND '40' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.80 NAME 'printJobConfiguration' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_NAME 'Print Job Configuration' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.79 NAME 'printerControl' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_NAME 'Printer Control' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.82 NAME 'privateKey' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NAME 'Private Key' X-NDS_NONREMOVABLE '1' X-NDS_HIDDEN '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.83 NAME 'Profile' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.84 NAME 'publicKey' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NAME 'Public Key' X-NDS_PUBLIC_READ '1' X-NDS_NONREMOVABLE '1' X-NDS_FILTERED_OPERATIONAL '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.85 NAME 'queue' SYNTAX 2.16.840.1.113719.1.1.5.1.25 X-NDS_NAME 'Queue' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.86 NAME 'queueDirectory' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{255} SINGLE-VALUE X-NDS_NAME 'Queue Directory' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '255' X-NDS_SERVER_READ '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.115 NAME 'Reference' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NEVER_SYNC '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' X-NDS_HIDDEN '1' X-NDS_FILTERED_REQUIRED '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.88 NAME 'Replica' SYNTAX 2.16.840.1.113719.1.1.5.1.16{64512} NO-USER-MODIFICATION USAGE directoryOperation X-NDS_PUBLIC_READ '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.89 NAME 'Resource' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_NONREMOVABLE '1' )", + "( 2.5.4.33 NAME 'roleOccupant' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_NAME 'Role Occupant' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.116 NAME 'higherPrivileges' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_NAME 'Higher Privileges' X-NDS_SERVER_READ '1' X-NDS_NAME_VALUE_ACCESS '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.92 NAME 'securityEquals' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_NAME 'Security Equals' X-NDS_SERVER_READ '1' X-NDS_NAME_VALUE_ACCESS '1' X-NDS_NONREMOVABLE '1' X-NDS_FILTERED_REQUIRED '1' )", + "( 2.5.4.34 NAME 'seeAlso' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_NAME 'See Also' X-NDS_NONREMOVABLE '1' )", + "( 2.5.4.5 NAME 'serialNumber' SYNTAX 1.3.6.1.4.1.1466.115.121.1.44{64} X-NDS_NAME 'Serial Number' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '64' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.95 NAME 'server' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_NAME 'Server' X-NDS_SERVER_READ '1' X-NDS_NONREMOVABLE '1' )", + "( 2.5.4.8 NAME ( 'st' 'stateOrProvinceName' ) SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{128} X-NDS_NAME 'S' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '128' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.98 NAME 'status' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NAME 'Status' X-NDS_PUBLIC_READ '1' X-NDS_NONREMOVABLE '1' X-NDS_FILTERED_OPERATIONAL '1' )", + "( 2.5.4.9 NAME 'street' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{128} X-NDS_NAME 'SA' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '128' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.102 NAME 'supportedTypefaces' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64} X-NDS_NAME 'Supported Typefaces' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '64' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.101 NAME 'supportedServices' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64} X-NDS_NAME 'Supported Services' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '64' X-NDS_NONREMOVABLE '1' )", + "( 2.5.4.4 NAME ( 'sn' 'surname' ) SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64} X-NDS_NAME 'Surname' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '64' X-NDS_PUBLIC_READ '1' X-NDS_NONREMOVABLE '1' )", + "( 2.5.4.20 NAME 'telephoneNumber' SYNTAX 1.3.6.1.4.1.1466.115.121.1.50{64512} X-NDS_NAME 'Telephone Number' X-NDS_NONREMOVABLE '1' )", + "( 2.5.4.12 NAME 'title' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64} X-NDS_NAME 'Title' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '64' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.111 NAME 'User' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_SERVER_READ '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.112 NAME 'Version' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64} SINGLE-VALUE X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '64' X-NDS_PUBLIC_READ '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.1 NAME 'accountBalance' SYNTAX 2.16.840.1.113719.1.1.5.1.22 SINGLE-VALUE X-NDS_NAME 'Account Balance' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.4 NAME 'allowUnlimitedCredit' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NAME 'Allow Unlimited Credit' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.118 NAME 'lowConvergenceResetTime' SYNTAX 1.3.6.1.4.1.1466.115.121.1.24 SINGLE-VALUE USAGE directoryOperation X-NDS_NAME 'Low Convergence Reset Time' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.54 NAME 'minimumAccountBalance' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NAME 'Minimum Account Balance' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.104 NAME 'lowConvergenceSyncInterval' SYNTAX 2.16.840.1.113719.1.1.5.1.27 SINGLE-VALUE X-NDS_NAME 'Low Convergence Sync Interval' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.21 NAME 'Device' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.53 NAME 'messageServer' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_NAME 'Message Server' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.34 NAME 'Language' SYNTAX 2.16.840.1.113719.1.1.5.1.6{64512} SINGLE-VALUE X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.100 NAME 'supportedConnections' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NAME 'Supported Connections' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.107 NAME 'typeCreatorMap' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_NAME 'Type Creator Map' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.108 NAME 'ndsUID' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NAME 'UID' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.24 NAME 'groupID' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NAME 'GID' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.110 NAME 'unknownBaseClass' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{32} SINGLE-VALUE USAGE directoryOperation X-NDS_NAME 'Unknown Base Class' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '32' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.87 NAME 'receivedUpTo' SYNTAX 2.16.840.1.113719.1.1.5.1.19 NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NAME 'Received Up To' X-NDS_PUBLIC_READ '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.33 NAME 'synchronizedUpTo' SYNTAX 2.16.840.1.113719.1.1.5.1.19 NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NAME 'Synchronized Up To' X-NDS_PUBLIC_READ '1' X-NDS_NEVER_SYNC '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.5 NAME 'authorityRevocation' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NAME 'Authority Revocation' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.13 NAME 'certificateRevocation' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NAME 'Certificate Revocation' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.17 NAME 'ndsCrossCertificatePair' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_NAME 'Cross Certificate Pair' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.37 NAME 'lockedByIntruder' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NAME 'Locked By Intruder' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.77 NAME 'printer' SYNTAX 2.16.840.1.113719.1.1.5.1.25 X-NDS_NAME 'Printer' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.20 NAME 'detectIntruder' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NAME 'Detect Intruder' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.38 NAME 'lockoutAfterDetection' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NAME 'Lockout After Detection' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.32 NAME 'intruderLockoutResetInterval' SYNTAX 2.16.840.1.113719.1.1.5.1.27 SINGLE-VALUE X-NDS_NAME 'Intruder Lockout Reset Interval' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.96 NAME 'serverHolds' SYNTAX 2.16.840.1.113719.1.1.5.1.26 X-NDS_NAME 'Server Holds' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.91 NAME 'sAPName' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{47} SINGLE-VALUE X-NDS_NAME 'SAP Name' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '47' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.113 NAME 'Volume' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.35 NAME 'lastLoginTime' SYNTAX 1.3.6.1.4.1.1466.115.121.1.24 SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NAME 'Last Login Time' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.81 NAME 'printServer' SYNTAX 2.16.840.1.113719.1.1.5.1.25 SINGLE-VALUE X-NDS_NAME 'Print Server' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.119 NAME 'nNSDomain' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{128} X-NDS_NAME 'NNS Domain' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '128' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.120 NAME 'fullName' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{127} X-NDS_NAME 'Full Name' X-NDS_UPPER_BOUND '127' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.121 NAME 'partitionControl' SYNTAX 2.16.840.1.113719.1.1.5.1.25 NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NAME 'Partition Control' X-NDS_PUBLIC_READ '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.122 NAME 'revision' SYNTAX 2.16.840.1.113719.1.1.5.1.22 SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NAME 'Revision' X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_SCHED_SYNC_NEVER '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.123 NAME 'certificateValidityInterval' SYNTAX 2.16.840.1.113719.1.1.5.1.27{4294967295} SINGLE-VALUE X-NDS_NAME 'Certificate Validity Interval' X-NDS_LOWER_BOUND '60' X-NDS_UPPER_BOUND '-1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.124 NAME 'externalSynchronizer' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_NAME 'External Synchronizer' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.125 NAME 'messagingDatabaseLocation' SYNTAX 2.16.840.1.113719.1.1.5.1.15{64512} SINGLE-VALUE X-NDS_NAME 'Messaging Database Location' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.126 NAME 'messageRoutingGroup' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_NAME 'Message Routing Group' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.127 NAME 'messagingServer' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_NAME 'Messaging Server' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.128 NAME 'Postmaster' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.162 NAME 'mailboxLocation' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_NAME 'Mailbox Location' X-NDS_PUBLIC_READ '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.163 NAME 'mailboxID' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{8} SINGLE-VALUE X-NDS_NAME 'Mailbox ID' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '8' X-NDS_PUBLIC_READ '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.164 NAME 'externalName' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_NAME 'External Name' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.165 NAME 'securityFlags' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NAME 'Security Flags' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.166 NAME 'messagingServerType' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{32} SINGLE-VALUE X-NDS_NAME 'Messaging Server Type' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '32' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.167 NAME 'lastReferencedTime' SYNTAX 2.16.840.1.113719.1.1.5.1.19 SINGLE-VALUE USAGE directoryOperation X-NDS_NAME 'Last Referenced Time' X-NDS_NEVER_SYNC '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.5.4.42 NAME 'givenName' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{32} X-NDS_NAME 'Given Name' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '32' X-NDS_PUBLIC_READ '1' X-NDS_NONREMOVABLE '1' )", + "( 2.5.4.43 NAME 'initials' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{8} X-NDS_NAME 'Initials' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '8' X-NDS_PUBLIC_READ '1' X-NDS_NONREMOVABLE '1' )", + "( 2.5.4.44 NAME 'generationQualifier' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{8} SINGLE-VALUE X-NDS_NAME 'Generational Qualifier' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '8' X-NDS_PUBLIC_READ '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.171 NAME 'profileMembership' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_NAME 'Profile Membership' X-NDS_NAME_VALUE_ACCESS '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.172 NAME 'dsRevision' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NAME 'DS Revision' X-NDS_PUBLIC_READ '1' X-NDS_NONREMOVABLE '1' X-NDS_FILTERED_OPERATIONAL '1' )", + "( 2.16.840.1.113719.1.1.4.1.173 NAME 'supportedGateway' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{4096} X-NDS_NAME 'Supported Gateway' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '4096' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.174 NAME 'equivalentToMe' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_NAME 'Equivalent To Me' X-NDS_SERVER_READ '1' X-NDS_NONREMOVABLE '1' X-NDS_FILTERED_REQUIRED '1' )", + "( 2.16.840.1.113719.1.1.4.1.175 NAME 'replicaUpTo' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NAME 'Replica Up To' X-NDS_PUBLIC_READ '1' X-NDS_NEVER_SYNC '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.176 NAME 'partitionStatus' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NAME 'Partition Status' X-NDS_PUBLIC_READ '1' X-NDS_NEVER_SYNC '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.177 NAME 'permanentConfigParms' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_NAME 'Permanent Config Parms' X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.178 NAME 'Timezone' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.179 NAME 'binderyRestrictionLevel' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE USAGE directoryOperation X-NDS_NAME 'Bindery Restriction Level' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.180 NAME 'transitiveVector' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NAME 'Transitive Vector' X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_SCHED_SYNC_NEVER '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.181 NAME 'T' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{32} X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '32' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.183 NAME 'purgeVector' SYNTAX 2.16.840.1.113719.1.1.5.1.19 NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NAME 'Purge Vector' X-NDS_PUBLIC_READ '1' X-NDS_NEVER_SYNC '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_SCHED_SYNC_NEVER '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.184 NAME 'synchronizationTolerance' SYNTAX 2.16.840.1.113719.1.1.5.1.19 USAGE directoryOperation X-NDS_NAME 'Synchronization Tolerance' X-NDS_PUBLIC_READ '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.185 NAME 'passwordManagement' SYNTAX 2.16.840.1.113719.1.1.5.1.0 SINGLE-VALUE USAGE directoryOperation X-NDS_NAME 'Password Management' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.186 NAME 'usedBy' SYNTAX 2.16.840.1.113719.1.1.5.1.15{64512} NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NAME 'Used By' X-NDS_SERVER_READ '1' X-NDS_NONREMOVABLE '1' X-NDS_FILTERED_REQUIRED '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.187 NAME 'Uses' SYNTAX 2.16.840.1.113719.1.1.5.1.15{64512} NO-USER-MODIFICATION USAGE directoryOperation X-NDS_SERVER_READ '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.500 NAME 'obituaryNotify' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NAME 'Obituary Notify' X-NDS_NONREMOVABLE '1' X-NDS_FILTERED_REQUIRED '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.501 NAME 'GUID' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{16} SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation X-NDS_LOWER_BOUND '16' X-NDS_UPPER_BOUND '16' X-NDS_PUBLIC_READ '1' X-NDS_NONREMOVABLE '1' X-NDS_FILTERED_REQUIRED '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.502 NAME 'otherGUID' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{16} USAGE directoryOperation X-NDS_NAME 'Other GUID' X-NDS_LOWER_BOUND '16' X-NDS_UPPER_BOUND '16' X-NDS_PUBLIC_READ '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.503 NAME 'auxiliaryClassFlag' SYNTAX 2.16.840.1.113719.1.1.5.1.0 SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NAME 'Auxiliary Class Flag' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.504 NAME 'unknownAuxiliaryClass' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{32} USAGE directoryOperation X-NDS_NAME 'Unknown Auxiliary Class' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '32' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 0.9.2342.19200300.100.1.1 NAME ( 'uid' 'userId' ) SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64} X-NDS_NAME 'uniqueID' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '64' X-NDS_PUBLIC_READ '1' X-NDS_NONREMOVABLE '1' )", + "( 0.9.2342.19200300.100.1.25 NAME 'dc' SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{64} X-NDS_NAME 'dc' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '64' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.507 NAME 'auxClassObjectClassBackup' SYNTAX 1.3.6.1.4.1.1466.115.121.1.38 NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NAME 'AuxClass Object Class Backup' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.508 NAME 'localReceivedUpTo' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NAME 'Local Received Up To' X-NDS_PUBLIC_READ '1' X-NDS_NEVER_SYNC '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.141.4.4 NAME 'federationControl' SYNTAX 2.16.840.1.113719.1.1.5.1.15{64512} USAGE directoryOperation X-NDS_NONREMOVABLE '1' X-NDS_FILTERED_REQUIRED '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.141.4.2 NAME 'federationSearchPath' SYNTAX 2.16.840.1.113719.1.1.5.1.6{64512} SINGLE-VALUE USAGE directoryOperation X-NDS_NONREMOVABLE '1' X-NDS_FILTERED_REQUIRED '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.141.4.3 NAME 'federationDNSName' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE USAGE directoryOperation X-NDS_PUBLIC_READ '1' X-NDS_NONREMOVABLE '1' X-NDS_FILTERED_REQUIRED '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.141.4.1 NAME 'federationBoundaryType' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NONREMOVABLE '1' X-NDS_FILTERED_REQUIRED '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.14.4.1.4 NAME 'DirXML-Associations' SYNTAX 2.16.840.1.113719.1.1.5.1.15{64512} X-NDS_NONREMOVABLE '1' X-NDS_FILTERED_REQUIRED '1' )", + "( 2.5.18.3 NAME 'creatorsName' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NONREMOVABLE '1' X-NDS_FILTERED_REQUIRED '1' X-NDS_READ_FILTERED '1' )", + "( 2.5.18.4 NAME 'modifiersName' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NONREMOVABLE '1' X-NDS_FILTERED_REQUIRED '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.300 NAME 'languageId' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.27.4.35 NAME 'ndsPredicate' SYNTAX 2.16.840.1.113719.1.1.5.1.12 X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.27.4.36 NAME 'ndsPredicateState' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.27.4.37 NAME 'ndsPredicateFlush' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.27.4.38 NAME 'ndsPredicateTimeout' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27{2147483647} SINGLE-VALUE X-NDS_UPPER_BOUND '2147483647' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.27.4.40 NAME 'ndsPredicateStatsDN' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.27.4.39 NAME 'ndsPredicateUseValues' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.601 NAME 'syncPanePoint' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation X-NDS_PUBLIC_READ '1' X-NDS_NEVER_SYNC '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.600 NAME 'syncWindowVector' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation X-NDS_PUBLIC_READ '1' X-NDS_NEVER_SYNC '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.602 NAME 'objectVersion' SYNTAX 2.16.840.1.113719.1.1.5.1.19 SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation X-NDS_PUBLIC_READ '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.27.4.52 NAME 'memberQueryURL' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_NAME 'memberQuery' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.302 NAME 'excludedMember' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.1.525 NAME 'auxClassCompatibility' SYNTAX 1.3.6.1.4.1.1466.115.121.1.38 NO-USER-MODIFICATION USAGE directoryOperation X-NDS_PUBLIC_READ '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.518 NAME 'ndsAgentPassword' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' X-NDS_HIDDEN '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.519 NAME 'ndsOperationCheckpoint' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} USAGE directoryOperation X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.520 NAME 'localReferral' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE USAGE directoryOperation X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.521 NAME 'treeReferral' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} USAGE directoryOperation X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.522 NAME 'schemaResetLock' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE USAGE directoryOperation X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.523 NAME 'modifiedACLEntry' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 USAGE directoryOperation X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.524 NAME 'monitoredConnection' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} USAGE directoryOperation X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.526 NAME 'localFederationBoundary' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE USAGE directoryOperation X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.527 NAME 'replicationFilter' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE USAGE directoryOperation X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.721 NAME 'ServerEBAEnabled' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NEVER_SYNC '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.716 NAME 'EBATreeConfiguration' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation X-NDS_PUBLIC_READ '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.722 NAME 'EBAPartitionConfiguration' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.723 NAME 'EBAServerConfiguration' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NEVER_SYNC '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' X-NDS_HIDDEN '1' )", + "( 2.16.840.1.113719.1.1.4.1.296 NAME 'loginActivationTime' SYNTAX 1.3.6.1.4.1.1466.115.121.1.24 SINGLE-VALUE X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.687 NAME 'UpdateInProgress' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.720 NAME 'dsContainerReadyAttrs' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.4.400.1 NAME 'edirSchemaFlagVersion' SYNTAX 2.16.840.1.113719.1.1.5.1.0 SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NONREMOVABLE '1' X-NDS_HIDDEN '1' X-NDS_READ_FILTERED '1' )", + "( 2.16.840.1.113719.1.1.4.1.512 NAME 'indexDefinition' SYNTAX 2.16.840.1.113719.1.1.5.1.6{64512} X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.1.4.1.513 NAME 'ndsStatusRepair' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.1.4.1.514 NAME 'ndsStatusExternalReference' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.1.4.1.515 NAME 'ndsStatusObituary' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.1.4.1.516 NAME 'ndsStatusSchema' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.1.4.1.517 NAME 'ndsStatusLimber' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.1.4.1.511 NAME 'authoritative' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113730.3.1.34 NAME 'ref' SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{64512} X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.1.4.1.546 NAME 'CachedAttrsOnExtRefs' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} X-NDS_SERVER_READ '1' )", + "( 2.16.840.1.113719.1.1.4.1.547 NAME 'ExtRefLastUpdatedTime' SYNTAX 2.16.840.1.113719.1.1.5.1.19 SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation X-NDS_PUBLIC_READ '1' X-NDS_NEVER_SYNC '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.1.4.688 NAME 'NCPKeyMaterialName' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.1.4.713 NAME 'UTF8LoginScript' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.1.4.714 NAME 'loginScriptCharset' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE )", + "( 2.16.840.1.113719.1.1.4.721 NAME 'NDSRightsToMonitor' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} X-NDS_NEVER_SYNC '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.1.4.1.1.192 NAME 'lDAPLogLevel' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27{32768} SINGLE-VALUE X-NDS_NAME 'LDAP Log Level' X-NDS_UPPER_BOUND '32768' )", + "( 2.16.840.1.113719.1.27.4.12 NAME 'lDAPUDPPort' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27{65535} SINGLE-VALUE X-NDS_NAME 'LDAP UDP Port' X-NDS_UPPER_BOUND '65535' )", + "( 2.16.840.1.113719.1.1.4.1.204 NAME 'lDAPLogFilename' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE X-NDS_NAME 'LDAP Log Filename' )", + "( 2.16.840.1.113719.1.1.4.1.205 NAME 'lDAPBackupLogFilename' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE X-NDS_NAME 'LDAP Backup Log Filename' )", + "( 2.16.840.1.113719.1.1.4.1.206 NAME 'lDAPLogSizeLimit' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27{4294967295} SINGLE-VALUE X-NDS_NAME 'LDAP Log Size Limit' X-NDS_LOWER_BOUND '2048' X-NDS_UPPER_BOUND '-1' )", + "( 2.16.840.1.113719.1.1.4.1.194 NAME 'lDAPSearchSizeLimit' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27{2147483647} SINGLE-VALUE X-NDS_NAME 'LDAP Search Size Limit' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '2147483647' )", + "( 2.16.840.1.113719.1.1.4.1.195 NAME 'lDAPSearchTimeLimit' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27{2147483647} SINGLE-VALUE X-NDS_NAME 'LDAP Search Time Limit' X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '2147483647' )", + "( 2.16.840.1.113719.1.1.4.1.207 NAME 'lDAPSuffix' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_NAME 'LDAP Suffix' )", + "( 2.16.840.1.113719.1.27.4.70 NAME 'ldapConfigVersion' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.27.4.14 NAME 'ldapReferral' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE X-NDS_NAME 'LDAP Referral' )", + "( 2.16.840.1.113719.1.27.4.73 NAME 'ldapDefaultReferralBehavior' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.27.4.23 NAME 'ldapSearchReferralUsage' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NAME 'LDAP:searchReferralUsage' )", + "( 2.16.840.1.113719.1.27.4.24 NAME 'lDAPOtherReferralUsage' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NAME 'LDAP:otherReferralUsage' )", + "( 2.16.840.1.113719.1.27.4.1 NAME 'ldapHostServer' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_NAME 'LDAP Host Server' )", + "( 2.16.840.1.113719.1.27.4.2 NAME 'ldapGroupDN' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_NAME 'LDAP Group' )", + "( 2.16.840.1.113719.1.27.4.3 NAME 'ldapTraceLevel' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27{32768} SINGLE-VALUE X-NDS_NAME 'LDAP Screen Level' X-NDS_UPPER_BOUND '32768' )", + "( 2.16.840.1.113719.1.27.4.4 NAME 'searchSizeLimit' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27{2147483647} SINGLE-VALUE X-NDS_UPPER_BOUND '2147483647' )", + "( 2.16.840.1.113719.1.27.4.5 NAME 'searchTimeLimit' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27{2147483647} SINGLE-VALUE X-NDS_UPPER_BOUND '2147483647' )", + "( 2.16.840.1.113719.1.27.4.6 NAME 'ldapServerBindLimit' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27{4294967295} SINGLE-VALUE X-NDS_NAME 'LDAP Server Bind Limit' X-NDS_UPPER_BOUND '-1' )", + "( 2.16.840.1.113719.1.27.4.7 NAME 'ldapServerIdleTimeout' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27{4294967295} SINGLE-VALUE X-NDS_NAME 'LDAP Server Idle Timeout' X-NDS_UPPER_BOUND '-1' )", + "( 2.16.840.1.113719.1.27.4.8 NAME 'ldapEnableTCP' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NAME 'LDAP Enable TCP' )", + "( 2.16.840.1.113719.1.27.4.10 NAME 'ldapEnableSSL' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NAME 'LDAP Enable SSL' )", + "( 2.16.840.1.113719.1.27.4.11 NAME 'ldapTCPPort' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27{65535} SINGLE-VALUE X-NDS_NAME 'LDAP TCP Port' X-NDS_UPPER_BOUND '65535' )", + "( 2.16.840.1.113719.1.27.4.13 NAME 'ldapSSLPort' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27{65535} SINGLE-VALUE X-NDS_NAME 'LDAP SSL Port' X-NDS_UPPER_BOUND '65535' )", + "( 2.16.840.1.113719.1.27.4.21 NAME 'filteredReplicaUsage' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.27.4.22 NAME 'ldapKeyMaterialName' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE X-NDS_NAME 'LDAP:keyMaterialName' )", + "( 2.16.840.1.113719.1.27.4.42 NAME 'extensionInfo' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.27.4.45 NAME 'nonStdClientSchemaCompatMode' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.27.4.46 NAME 'sslEnableMutualAuthentication' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.27.4.62 NAME 'ldapEnablePSearch' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.27.4.63 NAME 'ldapMaximumPSearchOperations' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.27.4.64 NAME 'ldapIgnorePSearchLimitsForEvents' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.27.4.65 NAME 'ldapTLSTrustedRootContainer' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 )", + "( 2.16.840.1.113719.1.27.4.66 NAME 'ldapEnableMonitorEvents' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.27.4.67 NAME 'ldapMaximumMonitorEventsLoad' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.27.4.68 NAME 'ldapTLSRequired' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.27.4.69 NAME 'ldapTLSVerifyClientCertificate' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.27.4.71 NAME 'ldapDerefAlias' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.27.4.72 NAME 'ldapNonStdAllUserAttrsMode' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.27.4.75 NAME 'ldapBindRestrictions' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.27.4.79 NAME 'ldapInterfaces' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113719.1.27.4.80 NAME 'ldapChainSecureRequired' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.27.4.82 NAME 'ldapStdCompliance' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.27.4.83 NAME 'ldapDerefAliasOnAuth' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.27.4.84 NAME 'ldapGeneralizedTime' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.27.4.85 NAME 'ldapPermissiveModify' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.27.4.86 NAME 'ldapSSLConfig' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.27.4.15 NAME 'ldapServerList' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_NAME 'LDAP Server List' )", + "( 2.16.840.1.113719.1.27.4.16 NAME 'ldapAttributeMap' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_NAME 'LDAP Attribute Map v11' )", + "( 2.16.840.1.113719.1.27.4.17 NAME 'ldapClassMap' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_NAME 'LDAP Class Map v11' )", + "( 2.16.840.1.113719.1.27.4.18 NAME 'ldapAllowClearTextPassword' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NAME 'LDAP Allow Clear Text Password' )", + "( 2.16.840.1.113719.1.27.4.19 NAME 'ldapAnonymousIdentity' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_NAME 'LDAP Anonymous Identity' )", + "( 2.16.840.1.113719.1.27.4.52 NAME 'ldapAttributeList' SYNTAX 2.16.840.1.113719.1.1.5.1.6{64512} )", + "( 2.16.840.1.113719.1.27.4.53 NAME 'ldapClassList' SYNTAX 2.16.840.1.113719.1.1.5.1.6{64512} )", + "( 2.16.840.1.113719.1.27.4.56 NAME 'transitionGroupDN' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.27.4.74 NAME 'ldapTransitionBackLink' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.27.4.78 NAME 'ldapLBURPNumWriterThreads' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.27.4.20 NAME 'ldapServerDN' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_NAME 'LDAP Server' )", + "( 0.9.2342.19200300.100.1.3 NAME 'mail' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} X-NDS_NAME 'Internet EMail Address' X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113730.3.1.3 NAME 'employeeNumber' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} X-NDS_NAME 'NSCP:employeeNumber' )", + "( 2.16.840.1.113719.1.27.4.76 NAME 'referralExcludeFilter' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113719.1.27.4.77 NAME 'referralIncludeFilter' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.5.4.36 NAME 'userCertificate' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_NAME 'userCertificate' X-NDS_PUBLIC_READ '1' )", + "( 2.5.4.37 NAME 'cACertificate' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_NAME 'cACertificate' X-NDS_PUBLIC_READ '1' )", + "( 2.5.4.40 NAME 'crossCertificatePair' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_NAME 'crossCertificatePair' X-NDS_PUBLIC_READ '1' )", + "( 2.5.4.58 NAME 'attributeCertificate' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_PUBLIC_READ '1' )", + "( 2.5.4.2 NAME 'knowledgeInformation' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{32768} X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '32768' )", + "( 2.5.4.14 NAME 'searchGuide' SYNTAX 1.3.6.1.4.1.1466.115.121.1.25{64512} X-NDS_NAME 'searchGuide' )", + "( 2.5.4.15 NAME 'businessCategory' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{128} X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '128' )", + "( 2.5.4.21 NAME 'telexNumber' SYNTAX 1.3.6.1.4.1.1466.115.121.1.52{64512} X-NDS_NAME 'telexNumber' )", + "( 2.5.4.22 NAME 'teletexTerminalIdentifier' SYNTAX 1.3.6.1.4.1.1466.115.121.1.51{64512} X-NDS_NAME 'teletexTerminalIdentifier' )", + "( 2.5.4.24 NAME 'x121Address' SYNTAX 1.3.6.1.4.1.1466.115.121.1.36{15} X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '15' )", + "( 2.5.4.25 NAME 'internationaliSDNNumber' SYNTAX 1.3.6.1.4.1.1466.115.121.1.36{16} X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '16' )", + "( 2.5.4.26 NAME 'registeredAddress' SYNTAX 1.3.6.1.4.1.1466.115.121.1.41{64512} X-NDS_NAME 'registeredAddress' )", + "( 2.5.4.27 NAME 'destinationIndicator' SYNTAX 1.3.6.1.4.1.1466.115.121.1.44{128} X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '128' )", + "( 2.5.4.28 NAME 'preferredDeliveryMethod' SYNTAX 1.3.6.1.4.1.1466.115.121.1.14{64512} SINGLE-VALUE X-NDS_NAME 'preferredDeliveryMethod' )", + "( 2.5.4.29 NAME 'presentationAddress' SYNTAX 1.3.6.1.4.1.1466.115.121.1.43{64512} SINGLE-VALUE X-NDS_NAME 'presentationAddress' )", + "( 2.5.4.30 NAME 'supportedApplicationContext' SYNTAX 1.3.6.1.4.1.1466.115.121.1.38{64512} X-NDS_NAME 'supportedApplicationContext' )", + "( 2.5.4.45 NAME 'x500UniqueIdentifier' SYNTAX 1.3.6.1.4.1.1466.115.121.1.6{64512} X-NDS_NAME 'x500UniqueIdentifier' )", + "( 2.5.4.46 NAME 'dnQualifier' SYNTAX 1.3.6.1.4.1.1466.115.121.1.44{64512} )", + "( 2.5.4.47 NAME 'enhancedSearchGuide' SYNTAX 1.3.6.1.4.1.1466.115.121.1.21{64512} X-NDS_NAME 'enhancedSearchGuide' )", + "( 2.5.4.48 NAME 'protocolInformation' SYNTAX 1.3.6.1.4.1.1466.115.121.1.42{64512} X-NDS_NAME 'protocolInformation' )", + "( 2.5.4.51 NAME 'houseIdentifier' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{32768} X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '32768' )", + "( 2.5.4.52 NAME 'supportedAlgorithms' SYNTAX 1.3.6.1.4.1.1466.115.121.1.49{64512} X-NDS_NAME 'supportedAlgorithms' )", + "( 2.5.4.54 NAME 'dmdName' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{32768} X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '32768' )", + "( 0.9.2342.19200300.100.1.6 NAME 'roomNumber' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 0.9.2342.19200300.100.1.38 NAME 'associatedName' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 )", + "( 2.5.4.49 NAME 'dn' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.3.4.1 NAME 'httpServerDN' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 )", + "( 2.16.840.1.113719.1.3.4.2 NAME 'httpHostServerDN' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.3.4.3 NAME 'httpThreadsPerCPU' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.3.4.4 NAME 'httpIOBufferSize' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.3.4.5 NAME 'httpRequestTimeout' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.3.4.6 NAME 'httpKeepAliveRequestTimeout' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.3.4.7 NAME 'httpSessionTimeout' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.3.4.8 NAME 'httpKeyMaterialObject' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.3.4.9 NAME 'httpTraceLevel' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.3.4.10 NAME 'httpAuthRequiresTLS' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.3.4.11 NAME 'httpDefaultClearPort' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.3.4.12 NAME 'httpDefaultTLSPort' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.3.4.13 NAME 'httpBindRestrictions' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.1.4.1.295 NAME 'emboxConfig' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE )", + "( 2.16.840.1.113719.1.54.4.1.1 NAME 'trusteesOfNewObject' SYNTAX 2.16.840.1.113719.1.1.5.1.17 X-NDS_NAME 'Trustees Of New Object' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.55.4.1.1 NAME 'newObjectSDSRights' SYNTAX 2.16.840.1.113719.1.1.5.1.17 X-NDS_NAME 'New Object's DS Rights' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.56.4.1.1 NAME 'newObjectSFSRights' SYNTAX 2.16.840.1.113719.1.1.5.1.15{64512} X-NDS_NAME 'New Object's FS Rights' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.57.4.1.1 NAME 'setupScript' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_NAME 'Setup Script' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.58.4.1.1 NAME 'runSetupScript' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NAME 'Run Setup Script' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.59.4.1.1 NAME 'membersOfTemplate' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_NAME 'Members Of Template' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.60.4.1.1 NAME 'volumeSpaceRestrictions' SYNTAX 2.16.840.1.113719.1.1.5.1.15{64512} X-NDS_NAME 'Volume Space Restrictions' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.61.4.1.1 NAME 'setPasswordAfterCreate' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NAME 'Set Password After Create' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.62.4.1.1 NAME 'homeDirectoryRights' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 X-NDS_NAME 'Home Directory Rights' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.63.4.1.1 NAME 'newObjectSSelfRights' SYNTAX 2.16.840.1.113719.1.1.5.1.17 X-NDS_NAME 'New Object's Self Rights' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.8.4.1 NAME 'digitalMeID' SYNTAX 2.16.840.1.113719.1.1.5.1.15{64512} SINGLE-VALUE )", + "( 2.16.840.1.113719.1.8.4.2 NAME 'assistant' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 )", + "( 2.16.840.1.113719.1.8.4.3 NAME 'assistantPhone' SYNTAX 1.3.6.1.4.1.1466.115.121.1.50{64512} )", + "( 2.16.840.1.113719.1.8.4.4 NAME 'city' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113719.1.8.4.5 NAME 'company' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 0.9.2342.19200300.100.1.43 NAME 'co' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113719.1.8.4.6 NAME 'directReports' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 )", + "( 0.9.2342.19200300.100.1.10 NAME 'manager' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 )", + "( 2.16.840.1.113719.1.8.4.7 NAME 'mailstop' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 0.9.2342.19200300.100.1.41 NAME 'mobile' SYNTAX 1.3.6.1.4.1.1466.115.121.1.50{64512} )", + "( 0.9.2342.19200300.100.1.40 NAME 'personalTitle' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 0.9.2342.19200300.100.1.42 NAME 'pager' SYNTAX 1.3.6.1.4.1.1466.115.121.1.50{64512} )", + "( 2.16.840.1.113719.1.8.4.8 NAME 'workforceID' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113719.1.8.4.9 NAME 'instantMessagingID' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113719.1.8.4.10 NAME 'preferredName' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 0.9.2342.19200300.100.1.7 NAME 'photo' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} )", + "( 2.16.840.1.113719.1.8.4.11 NAME 'jobCode' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113719.1.8.4.12 NAME 'siteLocation' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113719.1.8.4.13 NAME 'employeeStatus' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113730.3.1.4 NAME 'employeeType' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113719.1.8.4.14 NAME 'costCenter' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113719.1.8.4.15 NAME 'costCenterDescription' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113719.1.8.4.16 NAME 'tollFreePhoneNumber' SYNTAX 1.3.6.1.4.1.1466.115.121.1.50{64512} )", + "( 2.16.840.1.113719.1.8.4.17 NAME 'otherPhoneNumber' SYNTAX 1.3.6.1.4.1.1466.115.121.1.50{64512} )", + "( 2.16.840.1.113719.1.8.4.18 NAME 'managerWorkforceID' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113719.1.8.4.19 NAME 'jackNumber' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113730.3.1.2 NAME 'departmentNumber' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113719.1.8.4.20 NAME 'vehicleInformation' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113719.1.8.4.21 NAME 'accessCardNumber' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113719.1.8.4.32 NAME 'isManager' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.8.4.22 NAME 'homeCity' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113719.1.8.4.23 NAME 'homeEmailAddress' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 1.3.6.1.4.1.1466.101.120.31 NAME 'homeFax' SYNTAX 1.3.6.1.4.1.1466.115.121.1.50{64512} )", + "( 0.9.2342.19200300.100.1.20 NAME 'homePhone' SYNTAX 1.3.6.1.4.1.1466.115.121.1.50{64512} )", + "( 2.16.840.1.113719.1.8.4.24 NAME 'homeState' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 0.9.2342.19200300.100.1.39 NAME 'homePostalAddress' SYNTAX 1.3.6.1.4.1.1466.115.121.1.41{64512} )", + "( 2.16.840.1.113719.1.8.4.25 NAME 'homeZipCode' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113719.1.8.4.26 NAME 'personalMobile' SYNTAX 1.3.6.1.4.1.1466.115.121.1.50{64512} )", + "( 2.16.840.1.113719.1.8.4.27 NAME 'children' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113719.1.8.4.28 NAME 'spouse' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113719.1.8.4.29 NAME 'vendorName' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113719.1.8.4.30 NAME 'vendorAddress' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113719.1.8.4.31 NAME 'vendorPhoneNumber' SYNTAX 1.3.6.1.4.1.1466.115.121.1.50{64512} )", + "( 2.16.840.1.113719.1.1.4.1.303 NAME 'dgIdentity' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_NAME_VALUE_ACCESS '1' )", + "( 2.16.840.1.113719.1.1.4.1.304 NAME 'dgTimeOut' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.1.4.1.305 NAME 'dgAllowUnknown' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.1.4.1.306 NAME 'dgAllowDuplicates' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.1.4.1.546 NAME 'allowAliasToAncestor' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.39.4.1.1 NAME 'sASSecurityDN' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_NAME 'SAS:Security DN' X-NDS_SERVER_READ '1' )", + "( 2.16.840.1.113719.1.39.4.1.2 NAME 'sASServiceDN' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_NAME 'SAS:Service DN' X-NDS_SERVER_READ '1' )", + "( 2.16.840.1.113719.1.39.4.1.3 NAME 'sASSecretStore' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NAME 'SAS:SecretStore' )", + "( 2.16.840.1.113719.1.39.4.1.4 NAME 'sASSecretStoreKey' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE USAGE directoryOperation X-NDS_NAME 'SAS:SecretStore:Key' X-NDS_HIDDEN '1' )", + "( 2.16.840.1.113719.1.39.4.1.5 NAME 'sASSecretStoreData' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} USAGE directoryOperation X-NDS_NAME 'SAS:SecretStore:Data' X-NDS_HIDDEN '1' )", + "( 2.16.840.1.113719.1.39.4.1.6 NAME 'sASPKIStoreKeys' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} USAGE directoryOperation X-NDS_NAME 'SAS:PKIStore:Keys' X-NDS_HIDDEN '1' )", + "( 2.16.840.1.113719.1.48.4.1.1 NAME 'nDSPKIPublicKey' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_NAME 'NDSPKI:Public Key' X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.2 NAME 'nDSPKIPrivateKey' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_NAME 'NDSPKI:Private Key' )", + "( 2.16.840.1.113719.1.48.4.1.3 NAME 'nDSPKIPublicKeyCertificate' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_NAME 'NDSPKI:Public Key Certificate' X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.4 NAME 'nDSPKICertificateChain' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_NAME 'NDSPKI:Certificate Chain' X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.16 NAME 'nDSPKIPublicKeyEC' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_NAME 'NDSPKI:Public Key EC' X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.17 NAME 'nDSPKIPrivateKeyEC' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_NAME 'NDSPKI:Private Key EC' )", + "( 2.16.840.1.113719.1.48.4.1.18 NAME 'nDSPKIPublicKeyCertificateEC' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_NAME 'NDSPKI:Public Key Certificate EC' X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.19 NAME 'crossCertificatePairEC' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_NAME 'Cross Certificate Pair EC' X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.20 NAME 'nDSPKICertificateChainEC' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_NAME 'NDSPKI:Certificate Chain EC' X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.5 NAME 'nDSPKIParentCA' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE X-NDS_NAME 'NDSPKI:Parent CA' )", + "( 2.16.840.1.113719.1.48.4.1.6 NAME 'nDSPKIParentCADN' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_NAME 'NDSPKI:Parent CA DN' )", + "( 2.16.840.1.113719.1.48.4.1.20 NAME 'nDSPKISuiteBMode' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NAME 'NDSPKI:SuiteBMode' X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.7 NAME 'nDSPKIKeyFile' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_NAME 'NDSPKI:Key File' )", + "( 2.16.840.1.113719.1.48.4.1.8 NAME 'nDSPKISubjectName' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE X-NDS_NAME 'NDSPKI:Subject Name' )", + "( 2.16.840.1.113719.1.48.4.1.11 NAME 'nDSPKIGivenName' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE X-NDS_NAME 'NDSPKI:Given Name' )", + "( 2.16.840.1.113719.1.48.4.1.9 NAME 'nDSPKIKeyMaterialDN' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_NAME 'NDSPKI:Key Material DN' )", + "( 2.16.840.1.113719.1.48.4.1.10 NAME 'nDSPKITreeCADN' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_NAME 'NDSPKI:Tree CA DN' )", + "( 2.5.4.59 NAME 'cAECCertificate' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.12 NAME 'nDSPKIUserCertificateInfo' SYNTAX 2.16.840.1.113719.1.1.5.1.15{64512} X-NDS_NAME 'NDSPKI:userCertificateInfo' )", + "( 2.16.840.1.113719.1.48.4.1.13 NAME 'nDSPKITrustedRootCertificate' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_NAME 'NDSPKI:Trusted Root Certificate' X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.14 NAME 'nDSPKINotBefore' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE X-NDS_NAME 'NDSPKI:Not Before' X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.15 NAME 'nDSPKINotAfter' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE X-NDS_NAME 'NDSPKI:Not After' X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.101 NAME 'nDSPKISDKeyServerDN' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_NAME 'NDSPKI:SD Key Server DN' X-NDS_SERVER_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.102 NAME 'nDSPKISDKeyStruct' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_NAME 'NDSPKI:SD Key Struct' )", + "( 2.16.840.1.113719.1.48.4.1.103 NAME 'nDSPKISDKeyCert' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_NAME 'NDSPKI:SD Key Cert' )", + "( 2.16.840.1.113719.1.48.4.1.104 NAME 'nDSPKISDKeyID' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_NAME 'NDSPKI:SD Key ID' )", + "( 2.16.840.1.113719.1.39.4.1.105 NAME 'nDSPKIKeystore' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} USAGE directoryOperation X-NDS_NAME 'NDSPKI:Keystore' X-NDS_HIDDEN '1' )", + "( 2.16.840.1.113719.1.39.4.1.106 NAME 'ndspkiAdditionalRoots' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.31.4.2.3 NAME 'masvLabel' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.31.4.2.4 NAME 'masvProposedLabel' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.31.4.2.5 NAME 'masvDefaultRange' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.31.4.2.6 NAME 'masvAuthorizedRange' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.31.4.2.7 NAME 'masvDomainPolicy' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.31.4.1.8 NAME 'masvClearanceNames' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.31.4.1.9 NAME 'masvLabelNames' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.31.4.1.10 NAME 'masvLabelSecrecyLevelNames' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.31.4.1.11 NAME 'masvLabelSecrecyCategoryNames' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.31.4.1.12 NAME 'masvLabelIntegrityLevelNames' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.31.4.1.13 NAME 'masvLabelIntegrityCategoryNames' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.31.4.1.14 NAME 'masvPolicyUpdate' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.31.4.1.16 NAME 'masvNDSAttributeLabels' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.31.4.1.15 NAME 'masvPolicyDN' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.2 NAME 'sASLoginSequence' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} X-NDS_NAME 'SAS:Login Sequence' X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.8 NAME 'sASLoginPolicyUpdate' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NAME 'SAS:Login Policy Update' X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.38 NAME 'sasNMASProductOptions' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.74 NAME 'sasAuditConfiguration' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_SERVER_READ '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.14 NAME 'sASNDSPasswordWindow' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NAME 'SAS:NDS Password Window' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.15 NAME 'sASPolicyCredentials' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_NAME 'SAS:Policy Credentials' X-NDS_SERVER_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.16 NAME 'sASPolicyMethods' SYNTAX 2.16.840.1.113719.1.1.5.1.15{64512} X-NDS_NAME 'SAS:Policy Methods' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.17 NAME 'sASPolicyObjectVersion' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NAME 'SAS:Policy Object Version' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.18 NAME 'sASPolicyServiceSubtypes' SYNTAX 2.16.840.1.113719.1.1.5.1.15{64512} X-NDS_NAME 'SAS:Policy Service Subtypes' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.19 NAME 'sASPolicyServices' SYNTAX 2.16.840.1.113719.1.1.5.1.15{64512} X-NDS_NAME 'SAS:Policy Services' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.20 NAME 'sASPolicyUsers' SYNTAX 2.16.840.1.113719.1.1.5.1.15{64512} X-NDS_NAME 'SAS:Policy Users' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.21 NAME 'sASAllowNDSPasswordWindow' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NAME 'SAS:Allow NDS Password Window' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.9 NAME 'sASMethodIdentifier' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_NAME 'SAS:Method Identifier' X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.10 NAME 'sASMethodVendor' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE X-NDS_NAME 'SAS:Method Vendor' X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.11 NAME 'sASAdvisoryMethodGrade' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE X-NDS_NAME 'SAS:Advisory Method Grade' X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.12 NAME 'sASVendorSupport' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE X-NDS_NAME 'SAS:Vendor Support' X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.13 NAME 'sasCertificateSearchContainers' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.70 NAME 'sasNMASMethodConfigData' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.22 NAME 'sASLoginClientMethodNetWare' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_NAME 'SAS:Login Client Method NetWare' X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.23 NAME 'sASLoginServerMethodNetWare' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_NAME 'SAS:Login Server Method NetWare' X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.24 NAME 'sASLoginClientMethodWINNT' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_NAME 'SAS:Login Client Method WINNT' X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.25 NAME 'sASLoginServerMethodWINNT' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_NAME 'SAS:Login Server Method WINNT' X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.26 NAME 'sasLoginClientMethodSolaris' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.27 NAME 'sasLoginServerMethodSolaris' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.28 NAME 'sasLoginClientMethodLinux' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.29 NAME 'sasLoginServerMethodLinux' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.30 NAME 'sasLoginClientMethodTru64' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.31 NAME 'sasLoginServerMethodTru64' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.32 NAME 'sasLoginClientMethodAIX' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.33 NAME 'sasLoginServerMethodAIX' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.34 NAME 'sasLoginClientMethodHPUX' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.35 NAME 'sasLoginServerMethodHPUX' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.1000 NAME 'sasLoginClientMethods390' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.1001 NAME 'sasLoginServerMethods390' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.1002 NAME 'sasLoginClientMethodLinuxX64' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.1003 NAME 'sasLoginServerMethodLinuxX64' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.1004 NAME 'sasLoginClientMethodWinX64' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.1005 NAME 'sasLoginServerMethodWinX64' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.1006 NAME 'sasLoginClientMethodSolaris64' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.1007 NAME 'sasLoginServerMethodSolaris64' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.1008 NAME 'sasLoginClientMethodAIX64' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.1009 NAME 'sasLoginServerMethodAIX64' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.1011 NAME 'sasLoginServerMethodSolarisi386' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.1012 NAME 'sasLoginClientMethodSolarisi386' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.78 NAME 'sasUnsignedMethodModules' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.79 NAME 'sasServerModuleName' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.80 NAME 'sasServerModuleEntryPointName' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.81 NAME 'sasSASLMechanismName' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.82 NAME 'sasSASLMechanismEntryPointName' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.83 NAME 'sasClientModuleName' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.84 NAME 'sasClientModuleEntryPointName' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.36 NAME 'sASLoginMethodContainerDN' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_NAME 'SAS:Login Method Container DN' X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.37 NAME 'sASLoginPolicyDN' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_NAME 'SAS:Login Policy DN' X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.63 NAME 'sasPostLoginMethodContainerDN' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.38 NAME 'rADIUSActiveConnections' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_NAME 'RADIUS:Active Connections' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.39 NAME 'rADIUSAgedInterval' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NAME 'RADIUS:Aged Interval' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.40 NAME 'rADIUSAttributeList' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_NAME 'RADIUS:Attribute List' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.41 NAME 'rADIUSAttributeLists' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_NAME 'RADIUS:Attribute Lists' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.42 NAME 'rADIUSClient' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_NAME 'RADIUS:Client' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.43 NAME 'rADIUSCommonNameResolution' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NAME 'RADIUS:Common Name Resolution' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.44 NAME 'rADIUSConcurrentLimit' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NAME 'RADIUS:Concurrent Limit' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.45 NAME 'rADIUSConnectionHistory' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_NAME 'RADIUS:Connection History' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.46 NAME 'rADIUSDASVersion' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NAME 'RADIUS:DAS Version' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.47 NAME 'rADIUSDefaultProfile' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE X-NDS_NAME 'RADIUS:Default Profile' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.48 NAME 'rADIUSDialAccessGroup' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_NAME 'RADIUS:Dial Access Group' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.49 NAME 'rADIUSEnableCommonNameLogin' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NAME 'RADIUS:Enable Common Name Login' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.50 NAME 'rADIUSEnableDialAccess' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NAME 'RADIUS:Enable Dial Access' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.51 NAME 'rADIUSInterimAcctingTimeout' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NAME 'RADIUS:Interim Accting Timeout' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.52 NAME 'rADIUSLookupContexts' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_NAME 'RADIUS:Lookup Contexts' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.53 NAME 'rADIUSMaxDASHistoryRecord' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NAME 'RADIUS:Max DAS History Record' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.54 NAME 'rADIUSMaximumHistoryRecord' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NAME 'RADIUS:Maximum History Record' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.55 NAME 'rADIUSPassword' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_NAME 'RADIUS:Password' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.56 NAME 'rADIUSPasswordPolicy' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NAME 'RADIUS:Password Policy' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.57 NAME 'rADIUSPrivateKey' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_NAME 'RADIUS:Private Key' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.58 NAME 'rADIUSProxyContext' SYNTAX 2.16.840.1.113719.1.1.5.1.15{64512} X-NDS_NAME 'RADIUS:Proxy Context' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.59 NAME 'rADIUSProxyDomain' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_NAME 'RADIUS:Proxy Domain' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.60 NAME 'rADIUSProxyTarget' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_NAME 'RADIUS:Proxy Target' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.61 NAME 'rADIUSPublicKey' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_NAME 'RADIUS:Public Key' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.62 NAME 'rADIUSServiceList' SYNTAX 2.16.840.1.113719.1.1.5.1.15{64512} X-NDS_NAME 'RADIUS:Service List' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.3 NAME 'sASLoginSecret' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_NAME 'SAS:Login Secret' X-NDS_SERVER_READ '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.4 NAME 'sASLoginSecretKey' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_NAME 'SAS:Login Secret Key' X-NDS_SERVER_READ '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.5 NAME 'sASEncryptionType' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NAME 'SAS:Encryption Type' X-NDS_SERVER_READ '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.6 NAME 'sASLoginConfiguration' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_NAME 'SAS:Login Configuration' X-NDS_SERVER_READ '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.7 NAME 'sASLoginConfigurationKey' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_NAME 'SAS:Login Configuration Key' X-NDS_SERVER_READ '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.73 NAME 'sasDefaultLoginSequence' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.64 NAME 'sasAuthorizedLoginSequences' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.69 NAME 'sasAllowableSubjectNames' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.71 NAME 'sasLoginFailureDelay' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_SERVER_READ '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.72 NAME 'sasMethodVersion' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_SERVER_READ '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.1010 NAME 'sasUpdateLoginInfo' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_SERVER_READ '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.1011 NAME 'sasOTPEnabled' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_SERVER_READ '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.1012 NAME 'sasOTPCounter' SYNTAX 2.16.840.1.113719.1.1.5.1.22 SINGLE-VALUE X-NDS_SERVER_READ '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.1013 NAME 'sasOTPLookAheadWindow' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_SERVER_READ '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.1014 NAME 'sasOTPDigits' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_SERVER_READ '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.1015 NAME 'sasOTPReSync' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_SERVER_READ '1' )", + "( 2.16.840.1.113719.1.39.42.1.0.1016 NAME 'sasUpdateLoginTimeInterval' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_SERVER_READ '1' )", + "( 2.16.840.1.113719.1.6.4.1 NAME 'snmpGroupDN' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.6.4.2 NAME 'snmpServerList' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 )", + "( 2.16.840.1.113719.1.6.4.3 NAME 'snmpTrapConfig' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE )", + "( 2.16.840.1.113719.1.6.4.4 NAME 'snmpTrapDescription' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE )", + "( 2.16.840.1.113719.1.6.4.5 NAME 'snmpTrapInterval' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.6.4.6 NAME 'snmpTrapDisable' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.1.4.1.528 NAME 'ndapPartitionPasswordMgmt' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.1.4.1.529 NAME 'ndapClassPasswordMgmt' SYNTAX 2.16.840.1.113719.1.1.5.1.0 X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.1.4.1.530 NAME 'ndapPasswordMgmt' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.1.4.1.537 NAME 'ndapPartitionLoginMgmt' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.1.4.1.538 NAME 'ndapClassLoginMgmt' SYNTAX 2.16.840.1.113719.1.1.5.1.0 X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.1.4.1.539 NAME 'ndapLoginMgmt' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.1 NAME 'nspmPasswordKey' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE USAGE directoryOperation X-NDS_HIDDEN '1' )", + "( 2.16.840.1.113719.1.39.43.4.2 NAME 'nspmPassword' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE USAGE directoryOperation X-NDS_HIDDEN '1' )", + "( 2.16.840.1.113719.1.39.43.4.3 NAME 'nspmDistributionPassword' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE USAGE directoryOperation X-NDS_HIDDEN '1' )", + "( 2.16.840.1.113719.1.39.43.4.4 NAME 'nspmPasswordHistory' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} USAGE directoryOperation X-NDS_HIDDEN '1' )", + "( 2.16.840.1.113719.1.39.43.4.5 NAME 'nspmAdministratorChangeCount' SYNTAX 2.16.840.1.113719.1.1.5.1.22 SINGLE-VALUE USAGE directoryOperation X-NDS_HIDDEN '1' )", + "( 2.16.840.1.113719.1.39.43.4.6 NAME 'nspmPasswordPolicyDN' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.7 NAME 'nspmPreviousDistributionPassword' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE USAGE directoryOperation X-NDS_HIDDEN '1' )", + "( 2.16.840.1.113719.1.39.43.4.8 NAME 'nspmDoNotExpirePassword' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE )", + "( 1.3.6.1.4.1.42.2.27.8.1.16 NAME 'pwdChangedTime' SYNTAX 1.3.6.1.4.1.1466.115.121.1.24 SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation )", + "( 1.3.6.1.4.1.42.2.27.8.1.17 NAME 'pwdAccountLockedTime' SYNTAX 1.3.6.1.4.1.1466.115.121.1.24 SINGLE-VALUE NO-USER-MODIFICATION USAGE directoryOperation )", + "( 1.3.6.1.4.1.42.2.27.8.1.19 NAME 'pwdFailureTime' SYNTAX 1.3.6.1.4.1.1466.115.121.1.24 NO-USER-MODIFICATION USAGE directoryOperation )", + "( 2.16.840.1.113719.1.39.43.4.100 NAME 'nspmConfigurationOptions' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.102 NAME 'nspmChangePasswordMessage' SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{64512} SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.103 NAME 'nspmPasswordHistoryLimit' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.104 NAME 'nspmPasswordHistoryExpiration' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 1.3.6.1.4.1.42.2.27.8.1.4 NAME 'pwdInHistory' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.105 NAME 'nspmMinPasswordLifetime' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.106 NAME 'nspmAdminsDoNotExpirePassword' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.107 NAME 'nspmPasswordACL' SYNTAX 2.16.840.1.113719.1.1.5.1.17 )", + "( 2.16.840.1.113719.1.39.43.4.200 NAME 'nspmMaximumLength' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.201 NAME 'nspmMinUpperCaseCharacters' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.202 NAME 'nspmMaxUpperCaseCharacters' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.203 NAME 'nspmMinLowerCaseCharacters' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.204 NAME 'nspmMaxLowerCaseCharacters' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.205 NAME 'nspmNumericCharactersAllowed' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.206 NAME 'nspmNumericAsFirstCharacter' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.207 NAME 'nspmNumericAsLastCharacter' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.208 NAME 'nspmMinNumericCharacters' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.209 NAME 'nspmMaxNumericCharacters' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.210 NAME 'nspmSpecialCharactersAllowed' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.211 NAME 'nspmSpecialAsFirstCharacter' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.212 NAME 'nspmSpecialAsLastCharacter' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.213 NAME 'nspmMinSpecialCharacters' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.214 NAME 'nspmMaxSpecialCharacters' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.215 NAME 'nspmMaxRepeatedCharacters' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.216 NAME 'nspmMaxConsecutiveCharacters' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.217 NAME 'nspmMinUniqueCharacters' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.218 NAME 'nspmDisallowedAttributeValues' SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{64512} X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.219 NAME 'nspmExcludeList' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.220 NAME 'nspmCaseSensitive' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.221 NAME 'nspmPolicyPrecedence' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.222 NAME 'nspmExtendedCharactersAllowed' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.223 NAME 'nspmExtendedAsFirstCharacter' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.224 NAME 'nspmExtendedAsLastCharacter' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.225 NAME 'nspmMinExtendedCharacters' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.226 NAME 'nspmMaxExtendedCharacters' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.227 NAME 'nspmUpperAsFirstCharacter' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.228 NAME 'nspmUpperAsLastCharacter' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.229 NAME 'nspmLowerAsFirstCharacter' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.230 NAME 'nspmLowerAsLastCharacter' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.231 NAME 'nspmComplexityRules' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.233 NAME 'nspmAD2K8Syntax' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.234 NAME 'nspmAD2K8maxViolation' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.235 NAME 'nspmXCharLimit' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.236 NAME 'nspmXCharHistoryLimit' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.237 NAME 'nspmUnicodeAllowed' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.238 NAME 'nspmNonAlphaCharactersAllowed' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.239 NAME 'nspmMinNonAlphaCharacters' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.240 NAME 'nspmMaxNonAlphaCharacters' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.241 NAME 'nspmGraceLoginHistoryLimit' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.300 NAME 'nspmPolicyAgentContainerDN' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.301 NAME 'nspmPolicyAgentNetWare' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.302 NAME 'nspmPolicyAgentWINNT' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.303 NAME 'nspmPolicyAgentSolaris' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.304 NAME 'nspmPolicyAgentLinux' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.305 NAME 'nspmPolicyAgentAIX' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.43.4.306 NAME 'nspmPolicyAgentHPUX' SYNTAX 1.3.6.1.4.1.1466.115.121.1.5 SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 0.9.2342.19200300.100.1.55 NAME 'audio' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} )", + "( 2.16.840.1.113730.3.1.1 NAME 'carLicense' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113730.3.1.241 NAME 'displayName' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 0.9.2342.19200300.100.1.60 NAME 'jpegPhoto' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} )", + "( 1.3.6.1.4.1.250.1.57 NAME 'labeledUri' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 0.9.2342.19200300.100.1.7 NAME 'ldapPhoto' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} )", + "( 2.16.840.1.113730.3.1.39 NAME 'preferredLanguage' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE )", + "( 0.9.2342.19200300.100.1.21 NAME 'secretary' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 )", + "( 2.16.840.1.113730.3.1.40 NAME 'userSMIMECertificate' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} )", + "( 2.16.840.1.113730.3.1.216 NAME 'userPKCS12' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} )", + "( 2.16.840.1.113719.1.12.4.1.0 NAME 'auditAEncryptionKey' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_NAME 'Audit:A Encryption Key' )", + "( 2.16.840.1.113719.1.12.4.2.0 NAME 'auditBEncryptionKey' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_NAME 'Audit:B Encryption Key' )", + "( 2.16.840.1.113719.1.12.4.3.0 NAME 'auditContents' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NAME 'Audit:Contents' )", + "( 2.16.840.1.113719.1.12.4.4.0 NAME 'auditType' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NAME 'Audit:Type' )", + "( 2.16.840.1.113719.1.12.4.5.0 NAME 'auditCurrentEncryptionKey' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_NAME 'Audit:Current Encryption Key' )", + "( 2.16.840.1.113719.1.12.4.6.0 NAME 'auditFileLink' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_NAME 'Audit:File Link' )", + "( 2.16.840.1.113719.1.12.4.7.0 NAME 'auditLinkList' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_NAME 'Audit:Link List' )", + "( 2.16.840.1.113719.1.12.4.8.0 NAME 'auditPath' SYNTAX 2.16.840.1.113719.1.1.5.1.15{64512} SINGLE-VALUE X-NDS_NAME 'Audit:Path' )", + "( 2.16.840.1.113719.1.12.4.9.0 NAME 'auditPolicy' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_NAME 'Audit:Policy' )", + "( 2.16.840.1.113719.1.38.4.1.1 NAME 'wANMANWANPolicy' SYNTAX 2.16.840.1.113719.1.1.5.1.13{64512} X-NDS_NAME 'WANMAN:WAN Policy' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.38.4.1.2 NAME 'wANMANLANAreaMembership' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_NAME 'WANMAN:LAN Area Membership' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.38.4.1.3 NAME 'wANMANCost' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_NAME 'WANMAN:Cost' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.38.4.1.4 NAME 'wANMANDefaultCost' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NAME 'WANMAN:Default Cost' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.135.4.30 NAME 'rbsAssignedRoles' SYNTAX 2.16.840.1.113719.1.1.5.1.25 )", + "( 2.16.840.1.113719.1.135.4.31 NAME 'rbsContent' SYNTAX 2.16.840.1.113719.1.1.5.1.25 )", + "( 2.16.840.1.113719.1.135.4.32 NAME 'rbsContentMembership' SYNTAX 2.16.840.1.113719.1.1.5.1.25 )", + "( 2.16.840.1.113719.1.135.4.33 NAME 'rbsEntryPoint' SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{64512} SINGLE-VALUE )", + "( 2.16.840.1.113719.1.135.4.34 NAME 'rbsMember' SYNTAX 2.16.840.1.113719.1.1.5.1.25 )", + "( 2.16.840.1.113719.1.135.4.35 NAME 'rbsOwnedCollections' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 )", + "( 2.16.840.1.113719.1.135.4.36 NAME 'rbsPath' SYNTAX 2.16.840.1.113719.1.1.5.1.25 )", + "( 2.16.840.1.113719.1.135.4.37 NAME 'rbsParameters' SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{64512} )", + "( 2.16.840.1.113719.1.135.4.38 NAME 'rbsTaskRights' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} )", + "( 2.16.840.1.113719.1.135.4.39 NAME 'rbsTrusteeOf' SYNTAX 2.16.840.1.113719.1.1.5.1.25 )", + "( 2.16.840.1.113719.1.135.4.40 NAME 'rbsType' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{256} SINGLE-VALUE X-NDS_LOWER_BOUND '1' X-NDS_UPPER_BOUND '256' )", + "( 2.16.840.1.113719.1.135.4.41 NAME 'rbsURL' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE )", + "( 2.16.840.1.113719.1.135.4.42 NAME 'rbsTaskTemplates' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} )", + "( 2.16.840.1.113719.1.135.4.43 NAME 'rbsTaskTemplatesURL' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE )", + "( 2.16.840.1.113719.1.135.4.44 NAME 'rbsGALabel' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE )", + "( 2.16.840.1.113719.1.135.4.45 NAME 'rbsPageMembership' SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{64512} )", + "( 2.16.840.1.113719.1.135.4.46 NAME 'rbsTargetObjectType' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113719.1.135.4.47 NAME 'rbsContext' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.135.4.48 NAME 'rbsXMLInfo' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE )", + "( 2.16.840.1.113719.1.135.4.51 NAME 'rbsAssignedRoles2' SYNTAX 2.16.840.1.113719.1.1.5.1.25 )", + "( 2.16.840.1.113719.1.135.4.52 NAME 'rbsOwnedCollections2' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 )", + "( 2.16.840.1.113719.1.1.4.1.540 NAME 'prSyncPolicyDN' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_SERVER_READ '1' )", + "( 2.16.840.1.113719.1.1.4.1.541 NAME 'prSyncAttributes' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} X-NDS_SERVER_READ '1' )", + "( 2.16.840.1.113719.1.1.4.1.542 NAME 'dsEncryptedReplicationConfig' SYNTAX 2.16.840.1.113719.1.1.5.1.19 )", + "( 2.16.840.1.113719.1.1.4.1.543 NAME 'encryptionPolicyDN' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.1.4.1.544 NAME 'attrEncryptionRequiresSecure' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.1.4.1.545 NAME 'attrEncryptionDefinition' SYNTAX 2.16.840.1.113719.1.1.5.1.6{64512} X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.48.4.1.16 NAME 'ndspkiCRLFileName' SYNTAX 2.16.840.1.113719.1.1.5.1.15{64512} SINGLE-VALUE X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.17 NAME 'ndspkiStatus' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.18 NAME 'ndspkiIssueTime' SYNTAX 1.3.6.1.4.1.1466.115.121.1.24 SINGLE-VALUE X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.19 NAME 'ndspkiNextIssueTime' SYNTAX 1.3.6.1.4.1.1466.115.121.1.24 SINGLE-VALUE X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.20 NAME 'ndspkiAttemptTime' SYNTAX 1.3.6.1.4.1.1466.115.121.1.24 SINGLE-VALUE X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.21 NAME 'ndspkiTimeInterval' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.22 NAME 'ndspkiCRLMaxProcessingInterval' SYNTAX 2.16.840.1.113719.1.1.5.1.27 SINGLE-VALUE X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.23 NAME 'ndspkiCRLNumber' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.24 NAME 'ndspkiDistributionPoints' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.25 NAME 'ndspkiCRLProcessData' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.26 NAME 'ndspkiCRLConfigurationDNList' SYNTAX 2.16.840.1.113719.1.1.5.1.15{64512} X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.27 NAME 'ndspkiCADN' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.28 NAME 'ndspkiCRLContainerDN' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.29 NAME 'ndspkiIssuedCertContainerDN' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.30 NAME 'ndspkiDistributionPointDN' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.31 NAME 'ndspkiCRLConfigurationDN' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.32 NAME 'ndspkiDirectory' SYNTAX 2.16.840.1.113719.1.1.5.1.15{64512} )", + "( 2.5.4.38 NAME 'authorityRevocationList' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 SINGLE-VALUE X-NDS_NAME 'ndspkiAuthorityRevocationList' X-NDS_PUBLIC_READ '1' )", + "( 2.5.4.39 NAME 'certificateRevocationList' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 SINGLE-VALUE X-NDS_NAME 'ndspkiCertificateRevocationList' X-NDS_PUBLIC_READ '1' )", + "( 2.5.4.53 NAME 'deltaRevocationList' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 SINGLE-VALUE X-NDS_NAME 'ndspkiDeltaRevocationList' X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.36 NAME 'ndspkiTrustedRootList' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.37 NAME 'ndspkiSecurityRightsLevel' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.48.4.1.38 NAME 'ndspkiKMOExport' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113719.1.48.4.1.39 NAME 'ndspkiCRLECConfigurationDNList' SYNTAX 2.16.840.1.113719.1.1.5.1.15{64512} X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.40 NAME 'ndspkiCRLType' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.41 NAME 'ndspkiCRLExtendValidity' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.42 NAME 'ndspkiDefaultRSAKeySize' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.43 NAME 'ndspkiDefaultECCurve' SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{64512} SINGLE-VALUE X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.48.4.1.44 NAME 'ndspkiDefaultCertificateLife' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.7.4.1 NAME 'notfSMTPEmailHost' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.7.4.2 NAME 'notfSMTPEmailFrom' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.7.4.3 NAME 'notfSMTPEmailUserName' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.7.4.5 NAME 'notfMergeTemplateData' SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{64512} SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.7.4.6 NAME 'notfMergeTemplateSubject' SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{64512} SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.44.4.1 NAME 'nsimRequiredQuestions' SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{64512} SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.44.4.2 NAME 'nsimRandomQuestions' SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{64512} SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.44.4.3 NAME 'nsimNumberRandomQuestions' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.44.4.4 NAME 'nsimMinResponseLength' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.44.4.5 NAME 'nsimMaxResponseLength' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.44.4.6 NAME 'nsimForgottenLoginConfig' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.44.4.7 NAME 'nsimForgottenAction' SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{64512} SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.44.4.8 NAME 'nsimAssignments' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.44.4.9 NAME 'nsimChallengeSetDN' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.44.4.10 NAME 'nsimChallengeSetGUID' SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{64512} SINGLE-VALUE X-NDS_PUBLIC_READ '1' X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.44.4.11 NAME 'nsimPwdRuleEnforcement' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.39.44.4.12 NAME 'nsimHint' SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{64512} SINGLE-VALUE X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.39.44.4.13 NAME 'nsimPasswordReminder' SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{64512} SINGLE-VALUE )", + "( 2.16.840.1.113719.1.266.4.4 NAME 'sssProxyStoreKey' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE USAGE directoryOperation X-NDS_HIDDEN '1' )", + "( 2.16.840.1.113719.1.266.4.5 NAME 'sssProxyStoreSecrets' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} USAGE directoryOperation X-NDS_HIDDEN '1' )", + "( 2.16.840.1.113719.1.266.4.6 NAME 'sssActiveServerList' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} )", + "( 2.16.840.1.113719.1.266.4.7 NAME 'sssCacheRefreshInterval' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.266.4.8 NAME 'sssAdminList' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 )", + "( 2.16.840.1.113719.1.266.4.9 NAME 'sssAdminGALabel' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} SINGLE-VALUE )", + "( 2.16.840.1.113719.1.266.4.10 NAME 'sssEnableReadTimestamps' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.266.4.11 NAME 'sssDisableMasterPasswords' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.266.4.12 NAME 'sssEnableAdminAccess' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.266.4.13 NAME 'sssReadSecretPolicies' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} )", + "( 2.16.840.1.113719.1.266.4.14 NAME 'sssServerPolicyOverrideDN' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.1.4.1.531 NAME 'eDirCloneSource' SYNTAX 2.16.840.1.113719.1.1.5.1.15{64512} SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.1.4.1.532 NAME 'eDirCloneKeys' SYNTAX 1.3.6.1.4.1.1466.115.121.1.40{64512} NO-USER-MODIFICATION USAGE directoryOperation X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' X-NDS_HIDDEN '1' )", + "( 2.16.840.1.113719.1.1.4.1.533 NAME 'eDirCloneLock' SYNTAX 2.16.840.1.113719.1.1.5.1.15{64512} SINGLE-VALUE X-NDS_NOT_SCHED_SYNC_IMMEDIATE '1' )", + "( 2.16.840.1.113719.1.1.4.711 NAME 'groupMember' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 )", + "( 2.16.840.1.113719.1.1.4.712 NAME 'nestedConfig' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE )", + "( 2.16.840.1.113719.1.1.4.717 NAME 'xdasDSConfiguration' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113719.1.1.4.718 NAME 'xdasConfiguration' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113719.1.1.4.719 NAME 'xdasVersion' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27{32768} SINGLE-VALUE X-NDS_UPPER_BOUND '32768' )", + "( 2.16.840.1.113719.1.347.4.79 NAME 'NAuditInstrumentation' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113719.1.347.4.2 NAME 'NAuditLoggingServer' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_PUBLIC_READ '1' )", + "( 2.16.840.1.113719.1.1.4.724 NAME 'cefConfiguration' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15{64512} )", + "( 2.16.840.1.113719.1.1.4.725 NAME 'cefVersion' SYNTAX 1.3.6.1.4.1.1466.115.121.1.27{32768} SINGLE-VALUE X-NDS_UPPER_BOUND '32768' )" + ], + "createTimestamp": [], + "dITContentRules": [], + "dITStructureRules": [], + "ldapSyntaxes": [ + "( 1.3.6.1.4.1.1466.115.121.1.1 X-NDS_SYNTAX '9' )", + "( 1.3.6.1.4.1.1466.115.121.1.2 X-NDS_SYNTAX '9' )", + "( 1.3.6.1.4.1.1466.115.121.1.3 X-NDS_SYNTAX '3' )", + "( 1.3.6.1.4.1.1466.115.121.1.4 X-NDS_SYNTAX '9' )", + "( 1.3.6.1.4.1.1466.115.121.1.5 X-NDS_SYNTAX '21' )", + "( 1.3.6.1.4.1.1466.115.121.1.6 X-NDS_SYNTAX '9' )", + "( 1.3.6.1.4.1.1466.115.121.1.7 X-NDS_SYNTAX '7' )", + "( 2.16.840.1.113719.1.1.5.1.6 X-NDS_SYNTAX '6' )", + "( 1.3.6.1.4.1.1466.115.121.1.8 X-NDS_SYNTAX '9' )", + "( 1.3.6.1.4.1.1466.115.121.1.9 X-NDS_SYNTAX '9' )", + "( 1.3.6.1.4.1.1466.115.121.1.10 X-NDS_SYNTAX '9' )", + "( 2.16.840.1.113719.1.1.5.1.22 X-NDS_SYNTAX '22' )", + "( 1.3.6.1.4.1.1466.115.121.1.11 X-NDS_SYNTAX '3' )", + "( 1.3.6.1.4.1.1466.115.121.1.12 X-NDS_SYNTAX '1' )", + "( 1.3.6.1.4.1.1466.115.121.1.13 X-NDS_SYNTAX '9' )", + "( 1.3.6.1.4.1.1466.115.121.1.14 X-NDS_SYNTAX '9' )", + "( 1.3.6.1.4.1.1466.115.121.1.15 X-NDS_SYNTAX '3' )", + "( 1.3.6.1.4.1.1466.115.121.1.16 X-NDS_SYNTAX '3' )", + "( 1.3.6.1.4.1.1466.115.121.1.17 X-NDS_SYNTAX '3' )", + "( 1.3.6.1.4.1.1466.115.121.1.18 X-NDS_SYNTAX '9' )", + "( 1.3.6.1.4.1.1466.115.121.1.19 X-NDS_SYNTAX '9' )", + "( 1.3.6.1.4.1.1466.115.121.1.20 X-NDS_SYNTAX '9' )", + "( 1.3.6.1.4.1.1466.115.121.1.21 X-NDS_SYNTAX '9' )", + "( 1.3.6.1.4.1.1466.115.121.1.22 X-NDS_SYNTAX '11' )", + "( 1.3.6.1.4.1.1466.115.121.1.23 X-NDS_SYNTAX '9' )", + "( 1.3.6.1.4.1.1466.115.121.1.24 X-NDS_SYNTAX '24' )", + "( 1.3.6.1.4.1.1466.115.121.1.25 X-NDS_SYNTAX '9' )", + "( 1.3.6.1.4.1.1466.115.121.1.26 X-NDS_SYNTAX '2' )", + "( 1.3.6.1.4.1.1466.115.121.1.27 X-NDS_SYNTAX '8' )", + "( 1.3.6.1.4.1.1466.115.121.1.28 X-NDS_SYNTAX '9' )", + "( 1.2.840.113556.1.4.906 X-NDS_SYNTAX '29' )", + "( 1.3.6.1.4.1.1466.115.121.1.54 X-NDS_SYNTAX '3' )", + "( 1.3.6.1.4.1.1466.115.121.1.56 X-NDS_SYNTAX '9' )", + "( 1.3.6.1.4.1.1466.115.121.1.57 X-NDS_SYNTAX '3' )", + "( 1.3.6.1.4.1.1466.115.121.1.29 X-NDS_SYNTAX '9' )", + "( 1.3.6.1.4.1.1466.115.121.1.30 X-NDS_SYNTAX '3' )", + "( 1.3.6.1.4.1.1466.115.121.1.31 X-NDS_SYNTAX '3' )", + "( 1.3.6.1.4.1.1466.115.121.1.32 X-NDS_SYNTAX '3' )", + "( 1.3.6.1.4.1.1466.115.121.1.33 X-NDS_SYNTAX '3' )", + "( 1.3.6.1.4.1.1466.115.121.1.55 X-NDS_SYNTAX '9' )", + "( 1.3.6.1.4.1.1466.115.121.1.34 X-NDS_SYNTAX '3' )", + "( 1.3.6.1.4.1.1466.115.121.1.35 X-NDS_SYNTAX '3' )", + "( 2.16.840.1.113719.1.1.5.1.19 X-NDS_SYNTAX '19' )", + "( 1.3.6.1.4.1.1466.115.121.1.36 X-NDS_SYNTAX '5' )", + "( 2.16.840.1.113719.1.1.5.1.17 X-NDS_SYNTAX '17' )", + "( 1.3.6.1.4.1.1466.115.121.1.37 X-NDS_SYNTAX '3' )", + "( 2.16.840.1.113719.1.1.5.1.13 X-NDS_SYNTAX '13' )", + "( 1.3.6.1.4.1.1466.115.121.1.40 X-NDS_SYNTAX '9' )", + "( 1.3.6.1.4.1.1466.115.121.1.38 X-NDS_SYNTAX '20' )", + "( 1.3.6.1.4.1.1466.115.121.1.39 X-NDS_SYNTAX '3' )", + "( 1.3.6.1.4.1.1466.115.121.1.41 X-NDS_SYNTAX '18' )", + "( 1.3.6.1.4.1.1466.115.121.1.43 X-NDS_SYNTAX '9' )", + "( 1.3.6.1.4.1.1466.115.121.1.44 X-NDS_SYNTAX '4' )", + "( 1.3.6.1.4.1.1466.115.121.1.42 X-NDS_SYNTAX '9' )", + "( 2.16.840.1.113719.1.1.5.1.16 X-NDS_SYNTAX '16' )", + "( 1.3.6.1.4.1.1466.115.121.1.58 X-NDS_SYNTAX '9' )", + "( 1.3.6.1.4.1.1466.115.121.1.45 X-NDS_SYNTAX '9' )", + "( 1.3.6.1.4.1.1466.115.121.1.46 X-NDS_SYNTAX '9' )", + "( 1.3.6.1.4.1.1466.115.121.1.47 X-NDS_SYNTAX '9' )", + "( 1.3.6.1.4.1.1466.115.121.1.48 X-NDS_SYNTAX '9' )", + "( 1.3.6.1.4.1.1466.115.121.1.49 X-NDS_SYNTAX '9' )", + "( 2.16.840.1.113719.1.1.5.1.12 X-NDS_SYNTAX '12' )", + "( 2.16.840.1.113719.1.1.5.1.23 X-NDS_SYNTAX '23' )", + "( 2.16.840.1.113719.1.1.5.1.15 X-NDS_SYNTAX '15' )", + "( 2.16.840.1.113719.1.1.5.1.14 X-NDS_SYNTAX '14' )", + "( 1.3.6.1.4.1.1466.115.121.1.50 X-NDS_SYNTAX '10' )", + "( 1.3.6.1.4.1.1466.115.121.1.51 X-NDS_SYNTAX '9' )", + "( 1.3.6.1.4.1.1466.115.121.1.52 X-NDS_SYNTAX '9' )", + "( 2.16.840.1.113719.1.1.5.1.25 X-NDS_SYNTAX '25' )", + "( 1.3.6.1.4.1.1466.115.121.1.53 X-NDS_SYNTAX '9' )", + "( 2.16.840.1.113719.1.1.5.1.26 X-NDS_SYNTAX '26' )", + "( 2.16.840.1.113719.1.1.5.1.27 X-NDS_SYNTAX '27' )" + ], + "matchingRuleUse": [], + "matchingRules": [], + "modifyTimestamp": [ + "20190831135835Z" + ], + "nameForms": [], + "objectClass": [ + "top", + "subschema" + ], + "objectClasses": [ + "( 2.5.6.0 NAME 'Top' STRUCTURAL MUST objectClass MAY ( cAPublicKey $ cAPrivateKey $ certificateValidityInterval $ authorityRevocation $ lastReferencedTime $ equivalentToMe $ ACL $ backLink $ binderyProperty $ Obituary $ Reference $ revision $ ndsCrossCertificatePair $ certificateRevocation $ usedBy $ GUID $ otherGUID $ DirXML-Associations $ creatorsName $ modifiersName $ objectVersion $ auxClassCompatibility $ unknownBaseClass $ unknownAuxiliaryClass $ masvProposedLabel $ masvDefaultRange $ masvAuthorizedRange $ auditFileLink $ rbsAssignedRoles $ rbsOwnedCollections $ rbsAssignedRoles2 $ rbsOwnedCollections2 ) X-NDS_NONREMOVABLE '1' X-NDS_ACL_TEMPLATES '16#subtree#[Creator]#[Entry Rights]' )", + "( 1.3.6.1.4.1.42.2.27.1.2.1 NAME 'aliasObject' SUP Top STRUCTURAL MUST aliasedObjectName X-NDS_NAME 'Alias' X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' )", + "( 2.5.6.2 NAME 'Country' SUP Top STRUCTURAL MUST c MAY ( description $ searchGuide $ sssActiveServerList $ sssServerPolicyOverrideDN ) X-NDS_NAMING 'c' X-NDS_CONTAINMENT ( 'Top' 'treeRoot' 'domain' ) X-NDS_NONREMOVABLE '1' )", + "( 2.5.6.3 NAME 'Locality' SUP Top STRUCTURAL MAY ( description $ l $ seeAlso $ st $ street $ searchGuide $ sssActiveServerList $ sssServerPolicyOverrideDN ) X-NDS_NAMING ( 'l' 'st' ) X-NDS_CONTAINMENT ( 'Country' 'organizationalUnit' 'Locality' 'Organization' 'domain' ) X-NDS_NONREMOVABLE '1' )", + "( 2.5.6.4 NAME 'Organization' SUP ( ndsLoginProperties $ ndsContainerLoginProperties ) STRUCTURAL MUST o MAY ( description $ facsimileTelephoneNumber $ l $ loginScript $ eMailAddress $ physicalDeliveryOfficeName $ postalAddress $ postalCode $ postOfficeBox $ printJobConfiguration $ printerControl $ seeAlso $ st $ street $ telephoneNumber $ loginIntruderLimit $ intruderAttemptResetInterval $ detectIntruder $ lockoutAfterDetection $ intruderLockoutResetInterval $ nNSDomain $ mailboxLocation $ mailboxID $ x121Address $ registeredAddress $ destinationIndicator $ preferredDeliveryMethod $ telexNumber $ teletexTerminalIdentifier $ internationaliSDNNumber $ businessCategory $ searchGuide $ rADIUSAttributeLists $ rADIUSDefaultProfile $ rADIUSDialAccessGroup $ rADIUSEnableDialAccess $ rADIUSServiceList $ sssActiveServerList $ sssServerPolicyOverrideDN $ userPassword ) X-NDS_NAMING 'o' X-NDS_CONTAINMENT ( 'Top' 'treeRoot' 'Country' 'Locality' 'domain' ) X-NDS_NONREMOVABLE '1' X-NDS_ACL_TEMPLATES ( '2#entry#[Self]#loginScript' '2#entry#[Self]#printJobConfiguration') )", + "( 2.5.6.5 NAME 'organizationalUnit' SUP ( ndsLoginProperties $ ndsContainerLoginProperties ) STRUCTURAL MUST ou MAY ( description $ facsimileTelephoneNumber $ l $ loginScript $ eMailAddress $ physicalDeliveryOfficeName $ postalAddress $ postalCode $ postOfficeBox $ printJobConfiguration $ printerControl $ seeAlso $ st $ street $ telephoneNumber $ loginIntruderLimit $ intruderAttemptResetInterval $ detectIntruder $ lockoutAfterDetection $ intruderLockoutResetInterval $ nNSDomain $ mailboxLocation $ mailboxID $ x121Address $ registeredAddress $ destinationIndicator $ preferredDeliveryMethod $ telexNumber $ teletexTerminalIdentifier $ internationaliSDNNumber $ businessCategory $ searchGuide $ rADIUSAttributeLists $ rADIUSDefaultProfile $ rADIUSDialAccessGroup $ rADIUSEnableDialAccess $ rADIUSServiceList $ sssActiveServerList $ sssServerPolicyOverrideDN $ userPassword ) X-NDS_NAMING 'ou' X-NDS_CONTAINMENT ( 'Locality' 'Organization' 'organizationalUnit' 'domain' ) X-NDS_NAME 'Organizational Unit' X-NDS_NONREMOVABLE '1' X-NDS_ACL_TEMPLATES ( '2#entry#[Self]#loginScript' '2#entry#[Self]#printJobConfiguration') )", + "( 2.5.6.8 NAME 'organizationalRole' SUP Top STRUCTURAL MUST cn MAY ( description $ facsimileTelephoneNumber $ l $ eMailAddress $ ou $ physicalDeliveryOfficeName $ postalAddress $ postalCode $ postOfficeBox $ roleOccupant $ seeAlso $ st $ street $ telephoneNumber $ mailboxLocation $ mailboxID $ x121Address $ registeredAddress $ destinationIndicator $ preferredDeliveryMethod $ telexNumber $ teletexTerminalIdentifier $ internationaliSDNNumber ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'Organization' 'organizationalUnit' 'domain' ) X-NDS_NAME 'Organizational Role' X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' )", + "( 2.5.6.9 NAME ( 'groupOfNames' 'group' 'groupOfUniqueNames' ) SUP Top STRUCTURAL MUST cn MAY ( description $ l $ member $ ou $ o $ owner $ seeAlso $ groupID $ fullName $ eMailAddress $ mailboxLocation $ mailboxID $ Profile $ profileMembership $ loginScript $ businessCategory $ nspmPasswordPolicyDN ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'Organization' 'organizationalUnit' 'domain' ) X-NDS_NAME 'Group' X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' )", + "( 2.5.6.6 NAME 'Person' SUP ndsLoginProperties STRUCTURAL MUST ( cn $ sn ) MAY ( description $ seeAlso $ telephoneNumber $ fullName $ givenName $ initials $ generationQualifier $ uid $ assistant $ assistantPhone $ city $ st $ company $ co $ directReports $ manager $ mailstop $ mobile $ personalTitle $ pager $ workforceID $ instantMessagingID $ preferredName $ photo $ jobCode $ siteLocation $ employeeStatus $ employeeType $ costCenter $ costCenterDescription $ tollFreePhoneNumber $ otherPhoneNumber $ managerWorkforceID $ roomNumber $ jackNumber $ departmentNumber $ vehicleInformation $ accessCardNumber $ isManager $ userPassword ) X-NDS_NAMING ( 'cn' 'uid' ) X-NDS_CONTAINMENT ( 'Organization' 'organizationalUnit' 'domain' ) X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' )", + "( 2.5.6.7 NAME 'organizationalPerson' SUP Person STRUCTURAL MAY ( facsimileTelephoneNumber $ l $ eMailAddress $ ou $ physicalDeliveryOfficeName $ postalAddress $ postalCode $ postOfficeBox $ st $ street $ title $ mailboxLocation $ mailboxID $ uid $ mail $ employeeNumber $ destinationIndicator $ internationaliSDNNumber $ preferredDeliveryMethod $ registeredAddress $ teletexTerminalIdentifier $ telexNumber $ x121Address $ businessCategory $ roomNumber $ x500UniqueIdentifier ) X-NDS_NAMING ( 'cn' 'ou' 'uid' ) X-NDS_CONTAINMENT ( 'Organization' 'organizationalUnit' 'domain' ) X-NDS_NAME 'Organizational Person' X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113730.3.2.2 NAME 'inetOrgPerson' SUP organizationalPerson STRUCTURAL MAY ( groupMembership $ ndsHomeDirectory $ loginAllowedTimeMap $ loginDisabled $ loginExpirationTime $ loginGraceLimit $ loginGraceRemaining $ loginIntruderAddress $ loginIntruderAttempts $ loginIntruderResetTime $ loginMaximumSimultaneous $ loginScript $ loginTime $ networkAddressRestriction $ networkAddress $ passwordsUsed $ passwordAllowChange $ passwordExpirationInterval $ passwordExpirationTime $ passwordMinimumLength $ passwordRequired $ passwordUniqueRequired $ printJobConfiguration $ privateKey $ Profile $ publicKey $ securityEquals $ accountBalance $ allowUnlimitedCredit $ minimumAccountBalance $ messageServer $ Language $ ndsUID $ lockedByIntruder $ serverHolds $ lastLoginTime $ typeCreatorMap $ higherPrivileges $ printerControl $ securityFlags $ profileMembership $ Timezone $ sASServiceDN $ sASSecretStore $ sASSecretStoreKey $ sASSecretStoreData $ sASPKIStoreKeys $ userCertificate $ nDSPKIUserCertificateInfo $ nDSPKIKeystore $ rADIUSActiveConnections $ rADIUSAttributeLists $ rADIUSConcurrentLimit $ rADIUSConnectionHistory $ rADIUSDefaultProfile $ rADIUSDialAccessGroup $ rADIUSEnableDialAccess $ rADIUSPassword $ rADIUSServiceList $ audio $ businessCategory $ carLicense $ departmentNumber $ employeeNumber $ employeeType $ displayName $ givenName $ homePhone $ homePostalAddress $ initials $ jpegPhoto $ labeledUri $ mail $ manager $ mobile $ o $ pager $ ldapPhoto $ preferredLanguage $ roomNumber $ secretary $ uid $ userSMIMECertificate $ x500UniqueIdentifier $ userPKCS12 $ sssProxyStoreKey $ sssProxyStoreSecrets $ sssServerPolicyOverrideDN ) X-NDS_NAME 'User' X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' X-NDS_ACL_TEMPLATES ( '2#subtree#[Self]#[All Attributes Rights]' '6#entry#[Self]#loginScript' '1#subtree#[Root Template]#[Entry Rights]' '2#entry#[Public]#messageServer' '2#entry#[Root Template]#groupMembership' '6#entry#[Self]#printJobConfiguration' '2#entry#[Root Template]#networkAddress') )", + "( 2.5.6.14 NAME 'Device' SUP Top STRUCTURAL MUST cn MAY ( description $ l $ networkAddress $ ou $ o $ owner $ seeAlso $ serialNumber ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'Organization' 'organizationalUnit' 'domain' ) X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.6.1.4 NAME 'Computer' SUP Device STRUCTURAL MAY ( operator $ server $ status ) X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.6.1.17 NAME 'Printer' SUP Device STRUCTURAL MAY ( Cartridge $ printerConfiguration $ defaultQueue $ hostDevice $ printServer $ Memory $ networkAddressRestriction $ notify $ operator $ pageDescriptionLanguage $ queue $ status $ supportedTypefaces ) X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.6.1.21 NAME 'Resource' SUP Top ABSTRACT MUST cn MAY ( description $ hostResourceName $ l $ ou $ o $ seeAlso $ Uses ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'Organization' 'organizationalUnit' 'domain' ) X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.6.1.20 NAME 'Queue' SUP Resource STRUCTURAL MUST queueDirectory MAY ( Device $ operator $ server $ User $ networkAddress $ Volume $ hostServer ) X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' X-NDS_ACL_TEMPLATES '2#subtree#[Root Template]#[All Attributes Rights]' )", + "( 2.16.840.1.113719.1.1.6.1.3 NAME 'binderyQueue' SUP Queue STRUCTURAL MUST binderyType X-NDS_NAMING ( 'cn' 'binderyType' ) X-NDS_NAME 'Bindery Queue' X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' X-NDS_ACL_TEMPLATES '2#subtree#[Root Template]#[All Attributes Rights]' )", + "( 2.16.840.1.113719.1.1.6.1.26 NAME 'Volume' SUP Resource STRUCTURAL MUST hostServer MAY status X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' X-NDS_ACL_TEMPLATES ( '2#entry#[Root Template]#hostResourceName' '2#entry#[Root Template]#hostServer') )", + "( 2.16.840.1.113719.1.1.6.1.7 NAME 'directoryMap' SUP Resource STRUCTURAL MUST hostServer MAY path X-NDS_NAME 'Directory Map' X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.6.1.19 NAME 'Profile' SUP Top STRUCTURAL MUST ( cn $ loginScript ) MAY ( description $ l $ ou $ o $ seeAlso $ fullName ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'Organization' 'organizationalUnit' 'domain' ) X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.6.1.22 NAME 'Server' SUP Top ABSTRACT MUST cn MAY ( description $ hostDevice $ l $ ou $ o $ privateKey $ publicKey $ Resource $ seeAlso $ status $ User $ Version $ networkAddress $ accountBalance $ allowUnlimitedCredit $ minimumAccountBalance $ fullName $ securityEquals $ securityFlags $ Timezone $ ndapClassPasswordMgmt $ ndapClassLoginMgmt ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'Organization' 'organizationalUnit' 'domain' ) X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' X-NDS_ACL_TEMPLATES ( '2#entry#[Public]#networkAddress' '16#subtree#[Self]#[Entry Rights]') )", + "( 2.16.840.1.113719.1.1.6.1.10 NAME 'ncpServer' SUP Server STRUCTURAL MAY ( operator $ supportedServices $ messagingServer $ dsRevision $ permanentConfigParms $ ndsPredicateStatsDN $ languageId $ indexDefinition $ CachedAttrsOnExtRefs $ NCPKeyMaterialName $ NDSRightsToMonitor $ ldapServerDN $ httpServerDN $ emboxConfig $ sASServiceDN $ cACertificate $ cAECCertificate $ nDSPKIPublicKey $ nDSPKIPrivateKey $ nDSPKICertificateChain $ nDSPKIParentCADN $ nDSPKISDKeyID $ nDSPKISDKeyStruct $ snmpGroupDN $ wANMANWANPolicy $ wANMANLANAreaMembership $ wANMANCost $ wANMANDefaultCost $ encryptionPolicyDN $ eDirCloneSource $ eDirCloneLock $ xdasDSConfiguration $ xdasConfiguration $ xdasVersion $ NAuditLoggingServer $ NAuditInstrumentation $ cefConfiguration $ cefVersion ) X-NDS_NAME 'NCP Server' X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' X-NDS_ACL_TEMPLATES '2#entry#[Public]#messagingServer' )", + "( 2.16.840.1.113719.1.1.6.1.18 NAME 'printServer' SUP Server STRUCTURAL MAY ( operator $ printer $ sAPName ) X-NDS_NAME 'Print Server' X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' X-NDS_ACL_TEMPLATES '2#subtree#[Root Template]#[All Attributes Rights]' )", + "( 2.16.840.1.113719.1.1.6.1.31 NAME 'CommExec' SUP Server STRUCTURAL MAY networkAddressRestriction X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.6.1.2 NAME 'binderyObject' SUP Top STRUCTURAL MUST ( binderyObjectRestriction $ binderyType $ cn ) X-NDS_NAMING ( 'cn' 'binderyType' ) X-NDS_CONTAINMENT ( 'Organization' 'organizationalUnit' 'domain' ) X-NDS_NAME 'Bindery Object' X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.6.1.15 NAME 'Partition' AUXILIARY MAY ( Convergence $ partitionCreationTime $ Replica $ inheritedACL $ lowConvergenceSyncInterval $ receivedUpTo $ synchronizedUpTo $ authorityRevocation $ certificateRevocation $ cAPrivateKey $ cAPublicKey $ ndsCrossCertificatePair $ lowConvergenceResetTime $ highConvergenceSyncInterval $ partitionControl $ replicaUpTo $ partitionStatus $ transitiveVector $ purgeVector $ synchronizationTolerance $ obituaryNotify $ localReceivedUpTo $ federationControl $ syncPanePoint $ syncWindowVector $ EBAPartitionConfiguration $ authoritative $ allowAliasToAncestor $ sASSecurityDN $ masvLabel $ ndapPartitionPasswordMgmt $ ndapPartitionLoginMgmt $ prSyncPolicyDN $ dsEncryptedReplicationConfig ) X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.6.1.0 NAME 'aFPServer' SUP Server STRUCTURAL MAY ( serialNumber $ supportedConnections ) X-NDS_NAME 'AFP Server' X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.6.1.27 NAME 'messagingServer' SUP Server STRUCTURAL MAY ( messagingDatabaseLocation $ messageRoutingGroup $ Postmaster $ supportedServices $ messagingServerType $ supportedGateway ) X-NDS_NAME 'Messaging Server' X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' X-NDS_ACL_TEMPLATES ( '1#subtree#[Self]#[Entry Rights]' '2#subtree#[Self]#[All Attributes Rights]' '6#entry#[Self]#status' '2#entry#[Public]#messagingServerType' '2#entry#[Public]#messagingDatabaseLocation') )", + "( 2.16.840.1.113719.1.1.6.1.28 NAME 'messageRoutingGroup' SUP groupOfNames STRUCTURAL X-NDS_NAME 'Message Routing Group' X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' X-NDS_ACL_TEMPLATES ( '1#subtree#[Self]#[Entry Rights]' '2#subtree#[Self]#[All Attributes Rights]') )", + "( 2.16.840.1.113719.1.1.6.1.29 NAME 'externalEntity' SUP Top STRUCTURAL MUST cn MAY ( description $ seeAlso $ facsimileTelephoneNumber $ l $ eMailAddress $ ou $ physicalDeliveryOfficeName $ postalAddress $ postalCode $ postOfficeBox $ st $ street $ title $ externalName $ mailboxLocation $ mailboxID ) X-NDS_NAMING ( 'cn' 'ou' ) X-NDS_CONTAINMENT ( 'Organization' 'organizationalUnit' 'domain' ) X-NDS_NAME 'External Entity' X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' X-NDS_ACL_TEMPLATES '2#entry#[Public]#externalName' )", + "( 2.16.840.1.113719.1.1.6.1.30 NAME 'List' SUP Top STRUCTURAL MUST cn MAY ( description $ l $ member $ ou $ o $ eMailAddress $ mailboxLocation $ mailboxID $ owner $ seeAlso $ fullName ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'Organization' 'organizationalUnit' 'domain' ) X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' X-NDS_ACL_TEMPLATES '2#entry#[Root Template]#member' )", + "( 2.16.840.1.113719.1.1.6.1.32 NAME 'treeRoot' SUP Top STRUCTURAL MUST T MAY ( EBATreeConfiguration $ sssActiveServerList ) X-NDS_NAMING 'T' X-NDS_NAME 'Tree Root' X-NDS_NONREMOVABLE '1' )", + "( 0.9.2342.19200300.100.4.13 NAME 'domain' SUP ( Top $ ndsLoginProperties $ ndsContainerLoginProperties ) STRUCTURAL MUST dc MAY ( searchGuide $ o $ seeAlso $ businessCategory $ x121Address $ registeredAddress $ destinationIndicator $ preferredDeliveryMethod $ telexNumber $ teletexTerminalIdentifier $ telephoneNumber $ internationaliSDNNumber $ facsimileTelephoneNumber $ street $ postOfficeBox $ postalCode $ postalAddress $ physicalDeliveryOfficeName $ l $ associatedName $ description $ sssActiveServerList $ sssServerPolicyOverrideDN $ userPassword ) X-NDS_NAMING 'dc' X-NDS_CONTAINMENT ( 'Top' 'treeRoot' 'Country' 'Locality' 'Organization' 'organizationalUnit' 'domain' ) X-NDS_NONREMOVABLE '1' )", + "( 1.3.6.1.4.1.1466.344 NAME 'dcObject' AUXILIARY MUST dc X-NDS_NAMING 'dc' X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.6.1.33 NAME 'ndsLoginProperties' SUP Top ABSTRACT MAY ( groupMembership $ loginAllowedTimeMap $ loginDisabled $ loginExpirationTime $ loginGraceLimit $ loginGraceRemaining $ loginIntruderAddress $ loginIntruderAttempts $ loginIntruderResetTime $ loginMaximumSimultaneous $ loginScript $ loginTime $ networkAddressRestriction $ networkAddress $ passwordsUsed $ passwordAllowChange $ passwordExpirationInterval $ passwordExpirationTime $ passwordMinimumLength $ passwordRequired $ passwordUniqueRequired $ privateKey $ Profile $ publicKey $ securityEquals $ accountBalance $ allowUnlimitedCredit $ minimumAccountBalance $ Language $ lockedByIntruder $ serverHolds $ lastLoginTime $ higherPrivileges $ securityFlags $ profileMembership $ Timezone $ loginActivationTime $ UTF8LoginScript $ loginScriptCharset $ sASNDSPasswordWindow $ sASLoginSecret $ sASLoginSecretKey $ sASEncryptionType $ sASLoginConfiguration $ sASLoginConfigurationKey $ sasLoginFailureDelay $ sasDefaultLoginSequence $ sasAuthorizedLoginSequences $ sasAllowableSubjectNames $ sasUpdateLoginInfo $ sasOTPEnabled $ sasOTPCounter $ sasOTPDigits $ sasOTPReSync $ sasUpdateLoginTimeInterval $ ndapPasswordMgmt $ ndapLoginMgmt $ nspmPasswordKey $ nspmPassword $ pwdChangedTime $ pwdAccountLockedTime $ pwdFailureTime $ nspmDoNotExpirePassword $ nspmDistributionPassword $ nspmPreviousDistributionPassword $ nspmPasswordHistory $ nspmAdministratorChangeCount $ nspmPasswordPolicyDN $ nsimHint $ nsimPasswordReminder $ userPassword ) X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.141.6.1 NAME 'federationBoundary' AUXILIARY MUST federationBoundaryType MAY ( federationControl $ federationDNSName $ federationSearchPath ) X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.6.1.34 NAME 'ndsContainerLoginProperties' SUP Top ABSTRACT MAY ( loginIntruderLimit $ intruderAttemptResetInterval $ detectIntruder $ lockoutAfterDetection $ intruderLockoutResetInterval $ sasLoginFailureDelay $ sasDefaultLoginSequence $ sasAuthorizedLoginSequences $ sasUpdateLoginInfo $ sasOTPEnabled $ sasOTPDigits $ sasUpdateLoginTimeInterval $ ndapPasswordMgmt $ ndapLoginMgmt $ nspmPasswordPolicyDN ) X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.27.6.3 NAME 'ndsPredicateStats' SUP Top STRUCTURAL MUST ( cn $ ndsPredicateState $ ndsPredicateFlush ) MAY ( ndsPredicate $ ndsPredicateTimeout $ ndsPredicateUseValues ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'Country' 'Locality' 'Organization' 'organizationalUnit' 'domain' ) X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.6.400.1 NAME 'edirSchemaVersion' SUP Top ABSTRACT MAY edirSchemaFlagVersion X-NDS_NOT_CONTAINER '1' X-NDS_NONREMOVABLE '1' )", + "( 2.16.840.1.113719.1.1.6.1.47 NAME 'immediateSuperiorReference' AUXILIARY MAY ref X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.27.6.1 NAME 'ldapServer' SUP Top STRUCTURAL MUST cn MAY ( ldapHostServer $ ldapGroupDN $ ldapTraceLevel $ ldapServerBindLimit $ ldapServerIdleTimeout $ lDAPUDPPort $ lDAPSearchSizeLimit $ lDAPSearchTimeLimit $ lDAPLogLevel $ lDAPLogFilename $ lDAPBackupLogFilename $ lDAPLogSizeLimit $ Version $ searchSizeLimit $ searchTimeLimit $ ldapEnableTCP $ ldapTCPPort $ ldapEnableSSL $ ldapSSLPort $ ldapKeyMaterialName $ filteredReplicaUsage $ extensionInfo $ nonStdClientSchemaCompatMode $ sslEnableMutualAuthentication $ ldapEnablePSearch $ ldapMaximumPSearchOperations $ ldapIgnorePSearchLimitsForEvents $ ldapTLSTrustedRootContainer $ ldapEnableMonitorEvents $ ldapMaximumMonitorEventsLoad $ ldapTLSRequired $ ldapTLSVerifyClientCertificate $ ldapConfigVersion $ ldapDerefAlias $ ldapNonStdAllUserAttrsMode $ ldapBindRestrictions $ ldapDefaultReferralBehavior $ ldapReferral $ ldapSearchReferralUsage $ lDAPOtherReferralUsage $ ldapLBURPNumWriterThreads $ ldapInterfaces $ ldapChainSecureRequired $ ldapStdCompliance $ ldapDerefAliasOnAuth $ ldapGeneralizedTime $ ldapPermissiveModify $ ldapSSLConfig ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'Country' 'Locality' 'organizationalUnit' 'Organization' 'domain' ) X-NDS_NAME 'LDAP Server' X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.27.6.2 NAME 'ldapGroup' SUP Top STRUCTURAL MUST cn MAY ( ldapReferral $ ldapServerList $ ldapAllowClearTextPassword $ ldapAnonymousIdentity $ lDAPSuffix $ ldapAttributeMap $ ldapClassMap $ ldapSearchReferralUsage $ lDAPOtherReferralUsage $ transitionGroupDN $ ldapAttributeList $ ldapClassList $ ldapConfigVersion $ Version $ ldapDefaultReferralBehavior $ ldapTransitionBackLink $ ldapSSLConfig $ referralIncludeFilter $ referralExcludeFilter ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'Country' 'Locality' 'organizationalUnit' 'Organization' 'domain' ) X-NDS_NAME 'LDAP Group' X-NDS_NOT_CONTAINER '1' )", + "( 2.5.6.22 NAME 'pkiCA' AUXILIARY MAY ( cACertificate $ certificateRevocationList $ authorityRevocationList $ crossCertificatePair $ attributeCertificate $ publicKey $ privateKey $ networkAddress $ loginTime $ lastLoginTime $ cAECCertificate $ crossCertificatePairEC ) X-NDS_NOT_CONTAINER '1' )", + "( 2.5.6.21 NAME 'pkiUser' AUXILIARY MAY userCertificate X-NDS_NOT_CONTAINER '1' )", + "( 2.5.6.15 NAME 'strongAuthenticationUser' AUXILIARY MAY userCertificate X-NDS_NOT_CONTAINER '1' )", + "( 2.5.6.11 NAME 'applicationProcess' SUP Top STRUCTURAL MUST cn MAY ( seeAlso $ ou $ l $ description ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'Country' 'Locality' 'organizationalUnit' 'Organization' 'domain' ) )", + "( 2.5.6.12 NAME 'applicationEntity' SUP Top STRUCTURAL MUST ( presentationAddress $ cn ) MAY ( supportedApplicationContext $ seeAlso $ ou $ o $ l $ description ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'Country' 'Locality' 'organizationalUnit' 'Organization' 'domain' ) )", + "( 2.5.6.13 NAME 'dSA' SUP applicationEntity STRUCTURAL MAY knowledgeInformation X-NDS_CONTAINMENT ( 'Country' 'Locality' 'organizationalUnit' 'Organization' 'domain' ) )", + "( 2.5.6.16 NAME 'certificationAuthority' AUXILIARY MUST ( authorityRevocationList $ certificateRevocationList $ cACertificate ) MAY crossCertificatePair X-NDS_NOT_CONTAINER '1' )", + "( 2.5.6.18 NAME 'userSecurityInformation' AUXILIARY MAY supportedAlgorithms X-NDS_NOT_CONTAINER '1' )", + "( 2.5.6.20 NAME 'dmd' SUP ndsLoginProperties AUXILIARY MUST dmdName MAY ( searchGuide $ seeAlso $ businessCategory $ x121Address $ registeredAddress $ destinationIndicator $ preferredDeliveryMethod $ telexNumber $ teletexTerminalIdentifier $ telephoneNumber $ internationaliSDNNumber $ facsimileTelephoneNumber $ street $ postOfficeBox $ postalCode $ postalAddress $ physicalDeliveryOfficeName $ l $ description $ userPassword ) X-NDS_NOT_CONTAINER '1' )", + "( 2.5.6.16.2 NAME 'certificationAuthority-V2' AUXILIARY MUST ( authorityRevocationList $ certificateRevocationList $ cACertificate ) MAY ( crossCertificatePair $ deltaRevocationList ) X-NDS_NAME 'certificationAuthorityVer2' X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.3.6.1 NAME 'httpServer' SUP Top STRUCTURAL MUST cn MAY ( httpHostServerDN $ httpThreadsPerCPU $ httpIOBufferSize $ httpRequestTimeout $ httpKeepAliveRequestTimeout $ httpSessionTimeout $ httpKeyMaterialObject $ httpTraceLevel $ httpAuthRequiresTLS $ httpDefaultClearPort $ httpDefaultTLSPort $ httpBindRestrictions ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'domain' 'Country' 'Locality' 'organizationalUnit' 'Organization' ) X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.64.6.1.1 NAME 'Template' SUP Top STRUCTURAL MUST cn MAY ( trusteesOfNewObject $ newObjectSDSRights $ newObjectSFSRights $ setupScript $ runSetupScript $ membersOfTemplate $ volumeSpaceRestrictions $ setPasswordAfterCreate $ homeDirectoryRights $ accountBalance $ allowUnlimitedCredit $ description $ eMailAddress $ facsimileTelephoneNumber $ groupMembership $ higherPrivileges $ ndsHomeDirectory $ l $ Language $ loginAllowedTimeMap $ loginDisabled $ loginExpirationTime $ loginGraceLimit $ loginMaximumSimultaneous $ loginScript $ mailboxID $ mailboxLocation $ member $ messageServer $ minimumAccountBalance $ networkAddressRestriction $ newObjectSSelfRights $ ou $ passwordAllowChange $ passwordExpirationInterval $ passwordExpirationTime $ passwordMinimumLength $ passwordRequired $ passwordUniqueRequired $ physicalDeliveryOfficeName $ postalAddress $ postalCode $ postOfficeBox $ Profile $ st $ street $ securityEquals $ securityFlags $ seeAlso $ telephoneNumber $ title $ assistant $ assistantPhone $ city $ company $ co $ manager $ managerWorkforceID $ mailstop $ siteLocation $ employeeType $ costCenter $ costCenterDescription $ tollFreePhoneNumber $ departmentNumber ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'organizationalUnit' 'Organization' ) X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.8.6.1 NAME 'homeInfo' AUXILIARY MAY ( homeCity $ homeEmailAddress $ homeFax $ homePhone $ homeState $ homePostalAddress $ homeZipCode $ personalMobile $ spouse $ children ) X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.8.6.2 NAME 'contingentWorker' AUXILIARY MAY ( vendorName $ vendorAddress $ vendorPhoneNumber ) X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.1.6.1.45 NAME 'dynamicGroup' SUP ( groupOfNames $ ndsLoginProperties ) STRUCTURAL MAY ( memberQueryURL $ excludedMember $ dgIdentity $ dgAllowUnknown $ dgTimeOut $ dgAllowDuplicates $ userPassword ) X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.1.6.1.46 NAME 'dynamicGroupAux' SUP ( groupOfNames $ ndsLoginProperties ) AUXILIARY MAY ( memberQueryURL $ excludedMember $ dgIdentity $ dgAllowUnknown $ dgTimeOut $ dgAllowDuplicates $ userPassword ) X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.39.6.1.1 NAME 'sASSecurity' SUP Top STRUCTURAL MUST cn MAY ( nDSPKITreeCADN $ masvPolicyDN $ sASLoginPolicyDN $ sASLoginMethodContainerDN $ sasPostLoginMethodContainerDN $ nspmPolicyAgentContainerDN ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'Top' 'treeRoot' 'Country' 'Organization' 'domain' ) X-NDS_NAME 'SAS:Security' )", + "( 2.16.840.1.113719.1.39.6.1.2 NAME 'sASService' SUP Resource STRUCTURAL MAY ( hostServer $ privateKey $ publicKey $ allowUnlimitedCredit $ fullName $ lastLoginTime $ lockedByIntruder $ loginAllowedTimeMap $ loginDisabled $ loginExpirationTime $ loginIntruderAddress $ loginIntruderAttempts $ loginIntruderResetTime $ loginMaximumSimultaneous $ loginTime $ networkAddress $ networkAddressRestriction $ notify $ operator $ owner $ path $ securityEquals $ securityFlags $ status $ Version $ nDSPKIKeyMaterialDN $ ndspkiKMOExport ) X-NDS_NAMING 'cn' X-NDS_NAME 'SAS:Service' X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.48.6.1.1 NAME 'nDSPKICertificateAuthority' SUP Top STRUCTURAL MUST cn MAY ( hostServer $ nDSPKIPublicKey $ nDSPKIPrivateKey $ nDSPKIPublicKeyCertificate $ nDSPKICertificateChain $ nDSPKICertificateChainEC $ nDSPKIParentCA $ nDSPKIParentCADN $ nDSPKISubjectName $ nDSPKIPublicKeyEC $ nDSPKIPrivateKeyEC $ nDSPKIPublicKeyCertificateEC $ crossCertificatePairEC $ nDSPKISuiteBMode $ cACertificate $ cAECCertificate $ ndspkiCRLContainerDN $ ndspkiIssuedCertContainerDN $ ndspkiCRLConfigurationDNList $ ndspkiCRLECConfigurationDNList $ ndspkiSecurityRightsLevel $ ndspkiDefaultRSAKeySize $ ndspkiDefaultECCurve $ ndspkiDefaultCertificateLife ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT 'sASSecurity' X-NDS_NAME 'NDSPKI:Certificate Authority' X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.48.6.1.2 NAME 'nDSPKIKeyMaterial' SUP Top STRUCTURAL MUST cn MAY ( hostServer $ nDSPKIKeyFile $ nDSPKIPrivateKey $ nDSPKIPublicKey $ nDSPKIPublicKeyCertificate $ nDSPKICertificateChain $ nDSPKISubjectName $ nDSPKIGivenName $ ndspkiAdditionalRoots $ nDSPKINotBefore $ nDSPKINotAfter ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'sASSecurity' 'Organization' 'organizationalUnit' 'domain' ) X-NDS_NAME 'NDSPKI:Key Material' X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.48.6.1.3 NAME 'nDSPKITrustedRoot' SUP Top STRUCTURAL MUST cn MAY ndspkiTrustedRootList X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'sASSecurity' 'Organization' 'organizationalUnit' 'Country' 'Locality' 'domain' ) X-NDS_NAME 'NDSPKI:Trusted Root' )", + "( 2.16.840.1.113719.1.48.6.1.4 NAME 'nDSPKITrustedRootObject' SUP Top STRUCTURAL MUST ( cn $ nDSPKITrustedRootCertificate ) MAY ( nDSPKISubjectName $ nDSPKINotBefore $ nDSPKINotAfter $ externalName $ givenName $ sn ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT 'nDSPKITrustedRoot' X-NDS_NAME 'NDSPKI:Trusted Root Object' X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.48.6.1.101 NAME 'nDSPKISDKeyAccessPartition' SUP Top STRUCTURAL MUST cn X-NDS_NAMING 'cn' X-NDS_CONTAINMENT 'sASSecurity' X-NDS_NAME 'NDSPKI:SD Key Access Partition' )", + "( 2.16.840.1.113719.1.48.6.1.102 NAME 'nDSPKISDKeyList' SUP Top STRUCTURAL MUST cn MAY ( nDSPKISDKeyServerDN $ nDSPKISDKeyStruct $ nDSPKISDKeyCert ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT 'nDSPKISDKeyAccessPartition' X-NDS_NAME 'NDSPKI:SD Key List' )", + "( 2.16.840.1.113719.1.31.6.2.1 NAME 'mASVSecurityPolicy' SUP Top STRUCTURAL MUST cn MAY ( description $ masvDomainPolicy $ masvPolicyUpdate $ masvClearanceNames $ masvLabelNames $ masvLabelSecrecyLevelNames $ masvLabelSecrecyCategoryNames $ masvLabelIntegrityLevelNames $ masvLabelIntegrityCategoryNames $ masvNDSAttributeLabels ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT 'sASSecurity' X-NDS_NAME 'MASV:Security Policy' X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.39.42.2.0.1 NAME 'sASLoginMethodContainer' SUP Top STRUCTURAL MUST cn MAY description X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'sASSecurity' 'Country' 'Locality' 'organizationalUnit' 'Organization' ) X-NDS_NAME 'SAS:Login Method Container' )", + "( 2.16.840.1.113719.1.39.42.2.0.4 NAME 'sASLoginPolicy' SUP Top STRUCTURAL MUST cn MAY ( description $ privateKey $ publicKey $ sASAllowNDSPasswordWindow $ sASPolicyCredentials $ sASPolicyMethods $ sASPolicyObjectVersion $ sASPolicyServiceSubtypes $ sASPolicyServices $ sASPolicyUsers $ sASLoginSequence $ sASLoginPolicyUpdate $ sasNMASProductOptions $ sasPolicyMethods $ sasPolicyServices $ sasPolicyUsers $ sasAllowNDSPasswordWindow $ sasLoginFailureDelay $ sasDefaultLoginSequence $ sasAuthorizedLoginSequences $ sasAuditConfiguration $ sasUpdateLoginInfo $ sasOTPEnabled $ sasOTPLookAheadWindow $ sasOTPDigits $ sasUpdateLoginTimeInterval $ nspmPasswordPolicyDN ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT 'sASSecurity' X-NDS_NAME 'SAS:Login Policy' X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.39.42.2.0.7 NAME 'sASNMASBaseLoginMethod' SUP Top ABSTRACT MUST cn MAY ( description $ sASLoginSecret $ sASLoginSecretKey $ sASEncryptionType $ sASLoginConfiguration $ sASLoginConfigurationKey $ sASMethodIdentifier $ sASMethodVendor $ sASVendorSupport $ sASAdvisoryMethodGrade $ sASLoginClientMethodNetWare $ sASLoginServerMethodNetWare $ sASLoginClientMethodWINNT $ sASLoginServerMethodWINNT $ sasCertificateSearchContainers $ sasNMASMethodConfigData $ sasMethodVersion $ sASLoginPolicyUpdate $ sasUnsignedMethodModules $ sasServerModuleName $ sasServerModuleEntryPointName $ sasSASLMechanismName $ sasSASLMechanismEntryPointName $ sasClientModuleName $ sasClientModuleEntryPointName $ sasLoginClientMethodSolaris $ sasLoginServerMethodSolaris $ sasLoginClientMethodLinux $ sasLoginServerMethodLinux $ sasLoginClientMethodTru64 $ sasLoginServerMethodTru64 $ sasLoginClientMethodAIX $ sasLoginServerMethodAIX $ sasLoginClientMethodHPUX $ sasLoginServerMethodHPUX $ sasLoginClientMethods390 $ sasLoginServerMethods390 $ sasLoginClientMethodLinuxX64 $ sasLoginServerMethodLinuxX64 $ sasLoginClientMethodWinX64 $ sasLoginServerMethodWinX64 $ sasLoginClientMethodSolaris64 $ sasLoginServerMethodSolaris64 $ sasLoginClientMethodSolarisi386 $ sasLoginServerMethodSolarisi386 $ sasLoginClientMethodAIX64 $ sasLoginServerMethodAIX64 ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT 'sASLoginMethodContainer' X-NDS_NAME 'SAS:NMAS Base Login Method' )", + "( 2.16.840.1.113719.1.39.42.2.0.8 NAME 'sASNMASLoginMethod' SUP sASNMASBaseLoginMethod STRUCTURAL X-NDS_NAME 'SAS:NMAS Login Method' )", + "( 2.16.840.1.113719.1.39.42.2.0.9 NAME 'rADIUSDialAccessSystem' SUP Top STRUCTURAL MUST cn MAY ( publicKey $ privateKey $ rADIUSAgedInterval $ rADIUSClient $ rADIUSCommonNameResolution $ rADIUSConcurrentLimit $ rADIUSDASVersion $ rADIUSEnableCommonNameLogin $ rADIUSEnableDialAccess $ rADIUSInterimAcctingTimeout $ rADIUSLookupContexts $ rADIUSMaxDASHistoryRecord $ rADIUSMaximumHistoryRecord $ rADIUSPasswordPolicy $ rADIUSPrivateKey $ rADIUSProxyContext $ rADIUSProxyDomain $ rADIUSProxyTarget $ rADIUSPublicKey $ sASLoginConfiguration $ sASLoginConfigurationKey ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'Country' 'Locality' 'organizationalUnit' 'Organization' ) X-NDS_NAME 'RADIUS:Dial Access System' X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.39.42.2.0.10 NAME 'rADIUSProfile' SUP Top STRUCTURAL MUST cn MAY rADIUSAttributeList X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'Country' 'Locality' 'organizationalUnit' 'Organization' ) X-NDS_NAME 'RADIUS:Profile' X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.39.42.2.0.11 NAME 'sasPostLoginMethodContainer' SUP Top STRUCTURAL MUST cn MAY description X-NDS_NAMING 'cn' X-NDS_CONTAINMENT 'sASSecurity' )", + "( 2.16.840.1.113719.1.39.42.2.0.12 NAME 'sasPostLoginMethod' SUP Top STRUCTURAL MUST cn MAY ( description $ sASLoginSecret $ sASLoginSecretKey $ sASEncryptionType $ sASLoginConfiguration $ sASLoginConfigurationKey $ sASMethodIdentifier $ sASMethodVendor $ sASVendorSupport $ sASAdvisoryMethodGrade $ sASLoginClientMethodNetWare $ sASLoginServerMethodNetWare $ sASLoginClientMethodWINNT $ sASLoginServerMethodWINNT $ sasMethodVersion $ sASLoginPolicyUpdate $ sasUnsignedMethodModules $ sasServerModuleName $ sasServerModuleEntryPointName $ sasSASLMechanismName $ sasSASLMechanismEntryPointName $ sasClientModuleName $ sasClientModuleEntryPointName $ sasLoginClientMethodSolaris $ sasLoginServerMethodSolaris $ sasLoginClientMethodLinux $ sasLoginServerMethodLinux $ sasLoginClientMethodTru64 $ sasLoginServerMethodTru64 $ sasLoginClientMethodAIX $ sasLoginServerMethodAIX $ sasLoginClientMethodHPUX $ sasLoginServerMethodHPUX $ sasLoginClientMethods390 $ sasLoginServerMethods390 $ sasLoginClientMethodLinuxX64 $ sasLoginServerMethodLinuxX64 $ sasLoginClientMethodWinX64 $ sasLoginServerMethodWinX64 $ sasLoginClientMethodSolaris64 $ sasLoginServerMethodSolaris64 $ sasLoginClientMethodSolarisi386 $ sasLoginServerMethodSolarisi386 $ sasLoginClientMethodAIX64 $ sasLoginServerMethodAIX64 ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT 'sasPostLoginMethodContainer' )", + "( 2.16.840.1.113719.1.6.6.1 NAME 'snmpGroup' SUP Top STRUCTURAL MUST cn MAY ( Version $ snmpServerList $ snmpTrapDisable $ snmpTrapInterval $ snmpTrapDescription $ snmpTrapConfig ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'Country' 'Locality' 'domain' 'organizationalUnit' 'Organization' ) X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.39.43.6.2 NAME 'nspmPasswordPolicyContainer' SUP Top STRUCTURAL MUST cn MAY description X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'sASSecurity' 'Country' 'domain' 'Locality' 'Organization' 'organizationalUnit' ) )", + "( 2.16.840.1.113719.1.39.43.6.3 NAME 'nspmPolicyAgent' SUP Top STRUCTURAL MUST cn MAY ( description $ nspmPolicyAgentNetWare $ nspmPolicyAgentWINNT $ nspmPolicyAgentSolaris $ nspmPolicyAgentLinux $ nspmPolicyAgentAIX $ nspmPolicyAgentHPUX ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT 'nspmPasswordPolicyContainer' X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.39.43.6.1 NAME 'nspmPasswordPolicy' SUP Top STRUCTURAL MUST cn MAY ( description $ nspmPolicyPrecedence $ nspmConfigurationOptions $ nspmChangePasswordMessage $ passwordExpirationInterval $ loginGraceLimit $ nspmMinPasswordLifetime $ passwordUniqueRequired $ nspmPasswordHistoryLimit $ nspmPasswordHistoryExpiration $ passwordAllowChange $ passwordRequired $ passwordMinimumLength $ nspmMaximumLength $ nspmCaseSensitive $ nspmMinUpperCaseCharacters $ nspmMaxUpperCaseCharacters $ nspmMinLowerCaseCharacters $ nspmMaxLowerCaseCharacters $ nspmNumericCharactersAllowed $ nspmNumericAsFirstCharacter $ nspmNumericAsLastCharacter $ nspmMinNumericCharacters $ nspmMaxNumericCharacters $ nspmSpecialCharactersAllowed $ nspmSpecialAsFirstCharacter $ nspmSpecialAsLastCharacter $ nspmMinSpecialCharacters $ nspmMaxSpecialCharacters $ nspmMaxRepeatedCharacters $ nspmMaxConsecutiveCharacters $ nspmMinUniqueCharacters $ nspmDisallowedAttributeValues $ nspmExcludeList $ nspmExtendedCharactersAllowed $ nspmExtendedAsFirstCharacter $ nspmExtendedAsLastCharacter $ nspmMinExtendedCharacters $ nspmMaxExtendedCharacters $ nspmUpperAsFirstCharacter $ nspmUpperAsLastCharacter $ nspmLowerAsFirstCharacter $ nspmLowerAsLastCharacter $ nspmComplexityRules $ nspmAD2K8Syntax $ nspmAD2K8maxViolation $ nspmXCharLimit $ nspmXCharHistoryLimit $ nspmUnicodeAllowed $ nspmNonAlphaCharactersAllowed $ nspmMinNonAlphaCharacters $ nspmMaxNonAlphaCharacters $ pwdInHistory $ nspmAdminsDoNotExpirePassword $ nspmPasswordACL $ nsimChallengeSetDN $ nsimForgottenAction $ nsimForgottenLoginConfig $ nsimAssignments $ nsimChallengeSetGUID $ nsimPwdRuleEnforcement ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'nspmPasswordPolicyContainer' 'domain' 'Locality' 'Organization' 'organizationalUnit' 'Country' ) X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.39.43.6.4 NAME 'nspmPasswordAux' AUXILIARY MAY ( publicKey $ privateKey $ loginGraceLimit $ loginGraceRemaining $ passwordExpirationTime $ passwordRequired $ nspmPasswordKey $ nspmPassword $ nspmDistributionPassword $ nspmPreviousDistributionPassword $ nspmPasswordHistory $ nspmAdministratorChangeCount $ nspmPasswordPolicyDN $ pwdChangedTime $ pwdAccountLockedTime $ pwdFailureTime $ nspmDoNotExpirePassword ) X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.12.6.1.0 NAME 'auditFileObject' SUP Top STRUCTURAL MUST ( cn $ auditPolicy $ auditContents ) MAY ( description $ auditPath $ auditLinkList $ auditType $ auditCurrentEncryptionKey $ auditAEncryptionKey $ auditBEncryptionKey ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'Top' 'Country' 'Locality' 'Organization' 'organizationalUnit' 'treeRoot' 'domain' ) X-NDS_NAME 'Audit:File Object' X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.38.6.1.4 NAME 'wANMANLANArea' SUP Top STRUCTURAL MUST cn MAY ( description $ l $ member $ o $ ou $ owner $ seeAlso $ wANMANWANPolicy $ wANMANCost $ wANMANDefaultCost ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'Country' 'Locality' 'Organization' 'organizationalUnit' ) X-NDS_NAME 'WANMAN:LAN Area' X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.135.6.37.1 NAME 'rbsCollection' SUP Top STRUCTURAL MUST cn MAY ( owner $ description $ rbsXMLInfo ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'Country' 'Locality' 'organizationalUnit' 'Organization' 'domain' ) )", + "( 2.16.840.1.113719.1.135.6.30.1 NAME 'rbsExternalScope' SUP Top ABSTRACT MUST cn MAY ( rbsURL $ description $ rbsXMLInfo ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT 'rbsCollection' X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.135.6.31.1 NAME 'rbsModule' SUP Top STRUCTURAL MUST cn MAY ( rbsURL $ rbsPath $ rbsType $ description $ rbsXMLInfo ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT 'rbsCollection' )", + "( 2.16.840.1.113719.1.135.6.32.1 NAME 'rbsRole' SUP Top STRUCTURAL MUST cn MAY ( rbsContent $ rbsMember $ rbsTrusteeOf $ rbsGALabel $ rbsParameters $ description $ rbsXMLInfo ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT 'rbsCollection' )", + "( 2.16.840.1.113719.1.135.6.33.1 NAME 'rbsTask' SUP Top STRUCTURAL MUST cn MAY ( rbsContentMembership $ rbsType $ rbsTaskRights $ rbsEntryPoint $ rbsParameters $ rbsTaskTemplates $ rbsTaskTemplatesURL $ description $ rbsXMLInfo ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT 'rbsModule' X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.135.6.34.1 NAME 'rbsBook' SUP rbsTask STRUCTURAL MAY ( rbsTargetObjectType $ rbsPageMembership ) X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.135.6.35.1 NAME 'rbsScope' SUP groupOfNames STRUCTURAL MAY ( rbsContext $ rbsXMLInfo ) X-NDS_CONTAINMENT 'rbsRole' X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.135.6.45.1 NAME 'rbsCollection2' SUP Top STRUCTURAL MUST cn MAY ( rbsXMLInfo $ rbsParameters $ owner $ description ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'Country' 'Locality' 'organizationalUnit' 'Organization' 'domain' ) )", + "( 2.16.840.1.113719.1.135.6.38.1 NAME 'rbsExternalScope2' SUP Top ABSTRACT MUST cn MAY ( rbsXMLInfo $ description ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT 'rbsCollection2' X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.135.6.39.1 NAME 'rbsModule2' SUP Top STRUCTURAL MUST cn MAY ( rbsXMLInfo $ rbsPath $ rbsType $ description ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT 'rbsCollection2' )", + "( 2.16.840.1.113719.1.135.6.40.1 NAME 'rbsRole2' SUP Top STRUCTURAL MUST cn MAY ( rbsXMLInfo $ rbsContent $ rbsMember $ rbsTrusteeOf $ rbsParameters $ description ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT 'rbsCollection2' )", + "( 2.16.840.1.113719.1.135.6.41.1 NAME 'rbsTask2' SUP Top STRUCTURAL MUST cn MAY ( rbsXMLInfo $ rbsContentMembership $ rbsType $ rbsTaskRights $ rbsEntryPoint $ rbsParameters $ description ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT 'rbsModule2' X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.135.6.42.1 NAME 'rbsBook2' SUP rbsTask2 STRUCTURAL MAY ( rbsTargetObjectType $ rbsPageMembership ) X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.135.6.43.1 NAME 'rbsScope2' SUP groupOfNames STRUCTURAL MAY ( rbsContext $ rbsXMLInfo ) X-NDS_CONTAINMENT 'rbsRole2' X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.1.6.1.49 NAME 'prSyncPolicy' SUP Top STRUCTURAL MUST cn MAY prSyncAttributes X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'domain' 'Country' 'Locality' 'organizationalUnit' 'Organization' ) X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.1.6.1.50 NAME 'encryptionPolicy' SUP Top STRUCTURAL MUST cn MAY ( attrEncryptionDefinition $ attrEncryptionRequiresSecure ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'Country' 'Locality' 'domain' 'organizationalUnit' 'Organization' ) X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.48.6.1.5 NAME 'ndspkiContainer' SUP Top STRUCTURAL MUST cn X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'ndspkiContainer' 'sASSecurity' 'Organization' 'organizationalUnit' 'Country' 'Locality' 'nDSPKITrustedRoot' ) )", + "( 2.16.840.1.113719.1.48.6.1.6 NAME 'ndspkiCertificate' SUP Top STRUCTURAL MUST ( cn $ userCertificate ) MAY ( nDSPKISubjectName $ nDSPKINotBefore $ nDSPKINotAfter $ externalName $ givenName $ sn ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'sASSecurity' 'Organization' 'organizationalUnit' 'Country' 'Locality' 'ndspkiContainer' 'nDSPKITrustedRoot' ) X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.48.6.1.7 NAME 'ndspkiCRLConfiguration' SUP Top STRUCTURAL MUST cn MAY ( ndspkiCRLFileName $ ndspkiDirectory $ ndspkiStatus $ ndspkiIssueTime $ ndspkiNextIssueTime $ ndspkiAttemptTime $ ndspkiTimeInterval $ ndspkiCRLMaxProcessingInterval $ ndspkiCRLNumber $ ndspkiDistributionPoints $ ndspkiDistributionPointDN $ ndspkiCADN $ ndspkiCRLProcessData $ nDSPKIPublicKey $ nDSPKIPrivateKey $ nDSPKIPublicKeyCertificate $ nDSPKICertificateChain $ nDSPKIParentCA $ nDSPKIParentCADN $ nDSPKISubjectName $ cACertificate $ hostServer $ ndspkiCRLType $ ndspkiCRLExtendValidity ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT 'ndspkiContainer' )", + "( 2.5.6.19 NAME 'cRLDistributionPoint' SUP Top STRUCTURAL MUST cn MAY ( authorityRevocationList $ authorityRevocationList $ cACertificate $ certificateRevocationList $ certificateRevocationList $ crossCertificatePair $ deltaRevocationList $ deltaRevocationList $ ndspkiCRLConfigurationDN ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'Country' 'Locality' 'organizationalUnit' 'Organization' 'sASSecurity' 'domain' 'ndspkiCRLConfiguration' ) X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.7.6.1 NAME 'notfTemplateCollection' SUP Top STRUCTURAL MUST cn MAY ( notfSMTPEmailHost $ notfSMTPEmailFrom $ notfSMTPEmailUserName $ sASSecretStore ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT 'sASSecurity' )", + "( 2.16.840.1.113719.1.7.6.2 NAME 'notfMergeTemplate' SUP Top STRUCTURAL MUST cn MAY ( notfMergeTemplateData $ notfMergeTemplateSubject ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT 'notfTemplateCollection' X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.39.44.6.1 NAME 'nsimChallengeSet' SUP Top STRUCTURAL MUST cn MAY ( description $ nsimRequiredQuestions $ nsimRandomQuestions $ nsimNumberRandomQuestions $ nsimMinResponseLength $ nsimMaxResponseLength ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'nspmPasswordPolicyContainer' 'Country' 'domain' 'Locality' 'Organization' 'organizationalUnit' ) X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.266.6.1 NAME 'sssServerPolicies' SUP Top STRUCTURAL MUST cn MAY ( sssCacheRefreshInterval $ sssEnableReadTimestamps $ sssDisableMasterPasswords $ sssEnableAdminAccess $ sssAdminList $ sssAdminGALabel $ sssReadSecretPolicies ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT 'sASSecurity' )", + "( 2.16.840.1.113719.1.266.6.2 NAME 'sssServerPolicyOverride' SUP Top STRUCTURAL MUST cn MAY ( sssCacheRefreshInterval $ sssEnableReadTimestamps $ sssDisableMasterPasswords $ sssEnableAdminAccess $ sssAdminList $ sssAdminGALabel $ sssReadSecretPolicies ) X-NDS_NAMING 'cn' X-NDS_CONTAINMENT ( 'sssServerPolicies' 'Organization' 'organizationalUnit' 'Country' 'Locality' 'domain' ) X-NDS_NOT_CONTAINER '1' )", + "( 2.16.840.1.113719.1.1.6.1.91 NAME 'nestedGroupAux' AUXILIARY MAY ( groupMember $ excludedMember $ nestedConfig $ groupMembership ) X-NDS_NOT_CONTAINER '1' )" + ] + }, + "schema_entry": "cn=schema", + "type": "SchemaInfo" +} +""" + +edir_9_1_4_dsa_info = """ +{ + "raw": { + "abandonOps": [ + "0" + ], + "addEntryOps": [ + "0" + ], + "altServer": [], + "bindSecurityErrors": [ + "0" + ], + "chainings": [ + "0" + ], + "compareOps": [ + "0" + ], + "directoryTreeName": [ + "TEST_TREE" + ], + "dsaName": [ + "cn=MYSERVER,o=resources" + ], + "errors": [ + "0" + ], + "extendedOps": [ + "0" + ], + "inBytes": [ + "293" + ], + "inOps": [ + "3" + ], + "listOps": [ + "0" + ], + "modifyEntryOps": [ + "0" + ], + "modifyRDNOps": [ + "0" + ], + "namingContexts": [ + "" + ], + "oneLevelSearchOps": [ + "0" + ], + "outBytes": [ + "14" + ], + "readOps": [ + "1" + ], + "referralsReturned": [ + "0" + ], + "removeEntryOps": [ + "0" + ], + "repUpdatesIn": [ + "0" + ], + "repUpdatesOut": [ + "0" + ], + "searchOps": [ + "1" + ], + "securityErrors": [ + "0" + ], + "simpleAuthBinds": [ + "1" + ], + "strongAuthBinds": [ + "0" + ], + "subschemaSubentry": [ + "cn=schema" + ], + "supportedCapabilities": [], + "supportedControl": [ + "2.16.840.1.113719.1.27.101.6", + "2.16.840.1.113719.1.27.101.5", + "1.2.840.113556.1.4.319", + "2.16.840.1.113730.3.4.3", + "2.16.840.1.113730.3.4.2", + "2.16.840.1.113719.1.27.101.57", + "2.16.840.1.113719.1.27.103.7", + "2.16.840.1.113719.1.27.101.40", + "2.16.840.1.113719.1.27.101.41", + "1.2.840.113556.1.4.1413", + "1.2.840.113556.1.4.805", + "2.16.840.1.113730.3.4.18", + "1.2.840.113556.1.4.529" + ], + "supportedExtension": [ + "2.16.840.1.113719.1.148.100.1", + "2.16.840.1.113719.1.148.100.3", + "2.16.840.1.113719.1.148.100.5", + "2.16.840.1.113719.1.148.100.7", + "2.16.840.1.113719.1.148.100.9", + "2.16.840.1.113719.1.148.100.11", + "2.16.840.1.113719.1.148.100.13", + "2.16.840.1.113719.1.148.100.15", + "2.16.840.1.113719.1.148.100.17", + "2.16.840.1.113719.1.39.42.100.1", + "2.16.840.1.113719.1.39.42.100.3", + "2.16.840.1.113719.1.39.42.100.5", + "2.16.840.1.113719.1.39.42.100.7", + "2.16.840.1.113719.1.39.42.100.9", + "2.16.840.1.113719.1.39.42.100.11", + "2.16.840.1.113719.1.39.42.100.13", + "2.16.840.1.113719.1.39.42.100.15", + "2.16.840.1.113719.1.39.42.100.17", + "2.16.840.1.113719.1.39.42.100.19", + "2.16.840.1.113719.1.39.42.100.21", + "2.16.840.1.113719.1.39.42.100.23", + "2.16.840.1.113719.1.39.42.100.25", + "2.16.840.1.113719.1.39.42.100.27", + "2.16.840.1.113719.1.39.42.100.29", + "1.3.6.1.4.1.4203.1.11.1", + "2.16.840.1.113719.1.27.100.1", + "2.16.840.1.113719.1.27.100.3", + "2.16.840.1.113719.1.27.100.5", + "2.16.840.1.113719.1.27.100.7", + "2.16.840.1.113719.1.27.100.11", + "2.16.840.1.113719.1.27.100.13", + "2.16.840.1.113719.1.27.100.15", + "2.16.840.1.113719.1.27.100.17", + "2.16.840.1.113719.1.27.100.19", + "2.16.840.1.113719.1.27.100.21", + "2.16.840.1.113719.1.27.100.23", + "2.16.840.1.113719.1.27.100.25", + "2.16.840.1.113719.1.27.100.27", + "2.16.840.1.113719.1.27.100.29", + "2.16.840.1.113719.1.27.100.31", + "2.16.840.1.113719.1.27.100.33", + "2.16.840.1.113719.1.27.100.35", + "2.16.840.1.113719.1.27.100.37", + "2.16.840.1.113719.1.27.100.39", + "2.16.840.1.113719.1.27.100.41", + "2.16.840.1.113719.1.27.100.96", + "2.16.840.1.113719.1.27.100.98", + "2.16.840.1.113719.1.27.100.101", + "2.16.840.1.113719.1.27.100.103", + "2.16.840.1.113719.1.142.100.1", + "2.16.840.1.113719.1.142.100.4", + "2.16.840.1.113719.1.142.100.6", + "2.16.840.1.113719.1.27.100.9", + "2.16.840.1.113719.1.27.100.43", + "2.16.840.1.113719.1.27.100.45", + "2.16.840.1.113719.1.27.100.47", + "2.16.840.1.113719.1.27.100.49", + "2.16.840.1.113719.1.27.100.51", + "2.16.840.1.113719.1.27.100.53", + "2.16.840.1.113719.1.27.100.55", + "1.3.6.1.4.1.1466.20037", + "2.16.840.1.113719.1.27.100.79", + "2.16.840.1.113719.1.27.100.84", + "2.16.840.1.113719.1.27.103.1", + "2.16.840.1.113719.1.27.103.2" + ], + "supportedFeatures": [ + "1.3.6.1.4.1.4203.1.5.1", + "2.16.840.1.113719.1.27.99.1" + ], + "supportedGroupingTypes": [ + "2.16.840.1.113719.1.27.103.8" + ], + "supportedLDAPVersion": [ + "2", + "3" + ], + "supportedSASLMechanisms": [ + "NMAS_LOGIN" + ], + "unAuthBinds": [ + "0" + ], + "vendorName": [ + "NetIQ Corporation" + ], + "vendorVersion": [ + "LDAP Agent for NetIQ eDirectory 9.1.4 (40105.09)" + ], + "wholeSubtreeSearchOps": [ + "0" + ] + }, + "type": "DsaInfo" +} +""" \ No newline at end of file diff --git a/server/www/packages/packages-linux/x64/ldap3/protocol/schemas/slapd24.py b/server/www/packages/packages-linux/x64/ldap3/protocol/schemas/slapd24.py index 30e1795..23b86e8 100644 --- a/server/www/packages/packages-linux/x64/ldap3/protocol/schemas/slapd24.py +++ b/server/www/packages/packages-linux/x64/ldap3/protocol/schemas/slapd24.py @@ -5,7 +5,7 @@ # # Author: Giovanni Cannata # -# Copyright 2014 - 2018 Giovanni Cannata +# Copyright 2014 - 2019 Giovanni Cannata # # This file is part of ldap3. # diff --git a/server/www/packages/packages-linux/x64/ldap3/strategy/asyncStream.py b/server/www/packages/packages-linux/x64/ldap3/strategy/asyncStream.py index 7977d7e..cfd61c8 100644 --- a/server/www/packages/packages-linux/x64/ldap3/strategy/asyncStream.py +++ b/server/www/packages/packages-linux/x64/ldap3/strategy/asyncStream.py @@ -55,7 +55,11 @@ class AsyncStreamStrategy(AsyncStrategy): self.persistent_search_message_id = None self.streaming = False self.callback = None - self.events = Queue() + if ldap_connection.pool_size: + self.events = Queue(ldap_connection.pool_size) + else: + self.events = Queue() + del self._requests # remove _requests dict from Async Strategy def _start_listen(self): @@ -77,7 +81,6 @@ class AsyncStreamStrategy(AsyncStrategy): if not self._header_added and self.stream.tell() == 0: header = add_ldif_header(['-'])[0] self.stream.write(prepare_for_stream(header + self.line_separator + self.line_separator)) - ldif_lines = persistent_search_response_to_ldif(change) if self.stream and ldif_lines and not self.connection.closed: fragment = self.line_separator.join(ldif_lines) diff --git a/server/www/packages/packages-linux/x64/ldap3/strategy/asynchronous.py b/server/www/packages/packages-linux/x64/ldap3/strategy/asynchronous.py index 8ac79ee..f4da2b4 100644 --- a/server/www/packages/packages-linux/x64/ldap3/strategy/asynchronous.py +++ b/server/www/packages/packages-linux/x64/ldap3/strategy/asynchronous.py @@ -59,7 +59,7 @@ class AsyncStrategy(BaseStrategy): def run(self): """ - Wait for data on socket, compute the length of the message and wait for enough bytes to decode the message + Waits for data on socket, computes the length of the message and waits for enough bytes to decode the message Message are appended to strategy._responses """ unprocessed = b'' diff --git a/server/www/packages/packages-linux/x64/ldap3/strategy/base.py b/server/www/packages/packages-linux/x64/ldap3/strategy/base.py index 0506703..03ce56e 100644 --- a/server/www/packages/packages-linux/x64/ldap3/strategy/base.py +++ b/server/www/packages/packages-linux/x64/ldap3/strategy/base.py @@ -28,7 +28,6 @@ from struct import pack from platform import system from time import sleep from random import choice -from datetime import datetime from .. import SYNC, ANONYMOUS, get_config_parameter, BASE, ALL_ATTRIBUTES, ALL_OPERATIONAL_ATTRIBUTES, NO_ATTRIBUTES from ..core.results import DO_NOT_RAISE_EXCEPTIONS, RESULT_REFERRAL @@ -457,7 +456,7 @@ class BaseStrategy(object): """ message_type = ldap_message.getComponentByName('protocolOp').getName() component = ldap_message['protocolOp'].getComponent() - controls = ldap_message['controls'] + controls = ldap_message['controls'] if ldap_message['controls'].hasValue() else None if message_type == 'bindResponse': if not bytes(component['matchedDN']).startswith(b'NTLM'): # patch for microsoft ntlm authentication result = bind_response_to_dict(component) @@ -577,12 +576,12 @@ class BaseStrategy(object): return control_type, {'description': Oids.get(control_type, ''), 'criticality': criticality, 'value': control_value} @staticmethod - def decode_control_fast(control): + def decode_control_fast(control, from_server=True): """ decode control, return a 2-element tuple where the first element is the control oid and the second element is a dictionary with description (from Oids), criticality and decoded control value """ - control_type = str(to_unicode(control[0][3], from_server=True)) + control_type = str(to_unicode(control[0][3], from_server=from_server)) criticality = False control_value = None for r in control[1:]: @@ -701,13 +700,17 @@ class BaseStrategy(object): resp['attributes'][attr_type] = list() self.do_next_range_search(request, resp, attr_name) return True - def do_operation_on_referral(self, request, referrals): - if log_enabled(PROTOCOL): - log(PROTOCOL, 'following referral for <%s>', self.connection) + + def create_referral_connection(self, referrals): + referral_connection = None + selected_referral = None + cachekey = None valid_referral_list = self.valid_referral_list(referrals) if valid_referral_list: - preferred_referral_list = [referral for referral in valid_referral_list if referral['ssl'] == self.connection.server.ssl] - selected_referral = choice(preferred_referral_list) if preferred_referral_list else choice(valid_referral_list) + preferred_referral_list = [referral for referral in valid_referral_list if + referral['ssl'] == self.connection.server.ssl] + selected_referral = choice(preferred_referral_list) if preferred_referral_list else choice( + valid_referral_list) cachekey = (selected_referral['host'], selected_referral['port'] or self.connection.server.port, selected_referral['ssl']) if self.connection.use_referral_cache and cachekey in self.referral_cache: @@ -725,7 +728,8 @@ class BaseStrategy(object): local_certificate_file=self.connection.server.tls.certificate_file, validate=self.connection.server.tls.validate, version=self.connection.server.tls.version, - ca_certs_file=self.connection.server.tls.ca_certs_file) if selected_referral['ssl'] else None) + ca_certs_file=self.connection.server.tls.ca_certs_file) if + selected_referral['ssl'] else None) from ..core.connection import Connection @@ -758,6 +762,13 @@ class BaseStrategy(object): if self.connection.usage: self.connection._usage.referrals_followed += 1 + return selected_referral, referral_connection, cachekey + + def do_operation_on_referral(self, request, referrals): + if log_enabled(PROTOCOL): + log(PROTOCOL, 'following referral for <%s>', self.connection) + selected_referral, referral_connection, cachekey = self.create_referral_connection(referrals) + if selected_referral: if request['type'] == 'searchRequest': referral_connection.search(selected_referral['base'] or request['base'], selected_referral['filter'] or request['filter'], diff --git a/server/www/packages/packages-linux/x64/ldap3/strategy/ldifProducer.py b/server/www/packages/packages-linux/x64/ldap3/strategy/ldifProducer.py index 119e172..dfdde1b 100644 --- a/server/www/packages/packages-linux/x64/ldap3/strategy/ldifProducer.py +++ b/server/www/packages/packages-linux/x64/ldap3/strategy/ldifProducer.py @@ -101,6 +101,8 @@ class LdifProducerStrategy(BaseStrategy): self.connection.request = BaseStrategy.decode_request(message_type, request, controls) self.connection.request['controls'] = controls + if self._outstanding is None: + self._outstanding = dict() self._outstanding[message_id] = self.connection.request return message_id diff --git a/server/www/packages/packages-linux/x64/ldap3/strategy/mockBase.py b/server/www/packages/packages-linux/x64/ldap3/strategy/mockBase.py index f07c7c2..c933a59 100644 --- a/server/www/packages/packages-linux/x64/ldap3/strategy/mockBase.py +++ b/server/www/packages/packages-linux/x64/ldap3/strategy/mockBase.py @@ -724,12 +724,12 @@ class MockBaseStrategy(object): if extension[0] == '2.16.840.1.113719.1.27.100.31': # getBindDNRequest [NOVELL] result_code = 0 message = '' - response_name = '2.16.840.1.113719.1.27.100.32' # getBindDNResponse [NOVELL] + response_name = OctetString('2.16.840.1.113719.1.27.100.32') # getBindDNResponse [NOVELL] response_value = OctetString(self.bound) elif extension[0] == '1.3.6.1.4.1.4203.1.11.3': # WhoAmI [RFC4532] result_code = 0 message = '' - response_name = '1.3.6.1.4.1.4203.1.11.3' # WhoAmI [RFC4532] + response_name = OctetString('1.3.6.1.4.1.4203.1.11.3') # WhoAmI [RFC4532] response_value = OctetString(self.bound) break @@ -845,7 +845,6 @@ class MockBaseStrategy(object): attr_name = node.assertion['attr'] attr_value = node.assertion['value'] for candidate in candidates: - # if attr_name in self.connection.server.dit[candidate] and attr_value in self.connection.server.dit[candidate][attr_name]: if attr_name in self.connection.server.dit[candidate] and self.equal(candidate, attr_name, attr_value): node.matched.add(candidate) else: diff --git a/server/www/packages/packages-linux/x64/ldap3/strategy/mockSync.py b/server/www/packages/packages-linux/x64/ldap3/strategy/mockSync.py index b155781..4de6381 100644 --- a/server/www/packages/packages-linux/x64/ldap3/strategy/mockSync.py +++ b/server/www/packages/packages-linux/x64/ldap3/strategy/mockSync.py @@ -5,7 +5,7 @@ # # Author: Giovanni Cannata # -# Copyright 2014 - 2018 Giovanni Cannata +# Copyright 2014 - 2019 Giovanni Cannata # # This file is part of ldap3. # diff --git a/server/www/packages/packages-linux/x64/ldap3/strategy/restartable.py b/server/www/packages/packages-linux/x64/ldap3/strategy/restartable.py index 77ef4cd..3ffaac6 100644 --- a/server/www/packages/packages-linux/x64/ldap3/strategy/restartable.py +++ b/server/www/packages/packages-linux/x64/ldap3/strategy/restartable.py @@ -5,7 +5,7 @@ # # Author: Giovanni Cannata # -# Copyright 2014 - 2018 Giovanni Cannata +# Copyright 2014 - 2019 Giovanni Cannata # # This file is part of ldap3. # diff --git a/server/www/packages/packages-linux/x64/ldap3/strategy/reusable.py b/server/www/packages/packages-linux/x64/ldap3/strategy/reusable.py index d70c4d9..8d213c4 100644 --- a/server/www/packages/packages-linux/x64/ldap3/strategy/reusable.py +++ b/server/www/packages/packages-linux/x64/ldap3/strategy/reusable.py @@ -1,493 +1,495 @@ -""" -""" - -# Created on 2014.03.23 -# -# Author: Giovanni Cannata -# -# Copyright 2014 - 2018 Giovanni Cannata -# -# This file is part of ldap3. -# -# ldap3 is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# ldap3 is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with ldap3 in the COPYING and COPYING.LESSER files. -# If not, see . - -from datetime import datetime -from os import linesep -from threading import Thread, Lock -from time import sleep - -from .. import RESTARTABLE, get_config_parameter, AUTO_BIND_DEFAULT, AUTO_BIND_NONE, AUTO_BIND_NO_TLS, AUTO_BIND_TLS_AFTER_BIND, AUTO_BIND_TLS_BEFORE_BIND -from .base import BaseStrategy -from ..core.usage import ConnectionUsage -from ..core.exceptions import LDAPConnectionPoolNameIsMandatoryError, LDAPConnectionPoolNotStartedError, LDAPOperationResult, LDAPExceptionError, LDAPResponseTimeoutError -from ..utils.log import log, log_enabled, ERROR, BASIC -from ..protocol.rfc4511 import LDAP_MAX_INT - -TERMINATE_REUSABLE = 'TERMINATE_REUSABLE_CONNECTION' - -BOGUS_BIND = -1 -BOGUS_UNBIND = -2 -BOGUS_EXTENDED = -3 -BOGUS_ABANDON = -4 - -try: - from queue import Queue, Empty -except ImportError: # Python 2 - # noinspection PyUnresolvedReferences - from Queue import Queue, Empty - - -# noinspection PyProtectedMember -class ReusableStrategy(BaseStrategy): - """ - A pool of reusable SyncWaitRestartable connections with lazy behaviour and limited lifetime. - The connection using this strategy presents itself as a normal connection, but internally the strategy has a pool of - connections that can be used as needed. Each connection lives in its own thread and has a busy/available status. - The strategy performs the requested operation on the first available connection. - The pool of connections is instantiated at strategy initialization. - Strategy has two customizable properties, the total number of connections in the pool and the lifetime of each connection. - When lifetime is expired the connection is closed and will be open again when needed. - """ - pools = dict() - - def receiving(self): - raise NotImplementedError - - def _start_listen(self): - raise NotImplementedError - - def _get_response(self, message_id): - raise NotImplementedError - - def get_stream(self): - raise NotImplementedError - - def set_stream(self, value): - raise NotImplementedError - - # noinspection PyProtectedMember - class ConnectionPool(object): - """ - Container for the Connection Threads - """ - def __new__(cls, connection): - if connection.pool_name in ReusableStrategy.pools: # returns existing connection pool - pool = ReusableStrategy.pools[connection.pool_name] - if not pool.started: # if pool is not started remove it from the pools singleton and create a new onw - del ReusableStrategy.pools[connection.pool_name] - return object.__new__(cls) - if connection.pool_keepalive and pool.keepalive != connection.pool_keepalive: # change lifetime - pool.keepalive = connection.pool_keepalive - if connection.pool_lifetime and pool.lifetime != connection.pool_lifetime: # change keepalive - pool.lifetime = connection.pool_lifetime - if connection.pool_size and pool.pool_size != connection.pool_size: # if pool size has changed terminate and recreate the connections - pool.terminate_pool() - pool.pool_size = connection.pool_size - return pool - else: - return object.__new__(cls) - - def __init__(self, connection): - if not hasattr(self, 'workers'): - self.name = connection.pool_name - self.master_connection = connection - self.workers = [] - self.pool_size = connection.pool_size or get_config_parameter('REUSABLE_THREADED_POOL_SIZE') - self.lifetime = connection.pool_lifetime or get_config_parameter('REUSABLE_THREADED_LIFETIME') - self.keepalive = connection.pool_keepalive - self.request_queue = Queue() - self.open_pool = False - self.bind_pool = False - self.tls_pool = False - self._incoming = dict() - self.counter = 0 - self.terminated_usage = ConnectionUsage() if connection._usage else None - self.terminated = False - self.pool_lock = Lock() - ReusableStrategy.pools[self.name] = self - self.started = False - if log_enabled(BASIC): - log(BASIC, 'instantiated ConnectionPool: <%r>', self) - - def __str__(self): - s = 'POOL: ' + str(self.name) + ' - status: ' + ('started' if self.started else 'terminated') - s += ' - responses in queue: ' + str(len(self._incoming)) - s += ' - pool size: ' + str(self.pool_size) - s += ' - lifetime: ' + str(self.lifetime) - s += ' - keepalive: ' + str(self.keepalive) - s += ' - open: ' + str(self.open_pool) - s += ' - bind: ' + str(self.bind_pool) - s += ' - tls: ' + str(self.tls_pool) + linesep - s += 'MASTER CONN: ' + str(self.master_connection) + linesep - s += 'WORKERS:' - if self.workers: - for i, worker in enumerate(self.workers): - s += linesep + str(i).rjust(5) + ': ' + str(worker) - else: - s += linesep + ' no active workers in pool' - - return s - - def __repr__(self): - return self.__str__() - - def get_info_from_server(self): - for worker in self.workers: - with worker.worker_lock: - if not worker.connection.server.schema or not worker.connection.server.info: - worker.get_info_from_server = True - else: - worker.get_info_from_server = False - - def rebind_pool(self): - for worker in self.workers: - with worker.worker_lock: - worker.connection.rebind(self.master_connection.user, - self.master_connection.password, - self.master_connection.authentication, - self.master_connection.sasl_mechanism, - self.master_connection.sasl_credentials) - - def start_pool(self): - if not self.started: - self.create_pool() - for worker in self.workers: - with worker.worker_lock: - worker.thread.start() - self.started = True - self.terminated = False - if log_enabled(BASIC): - log(BASIC, 'worker started for pool <%s>', self) - return True - return False - - def create_pool(self): - if log_enabled(BASIC): - log(BASIC, 'created pool <%s>', self) - self.workers = [ReusableStrategy.PooledConnectionWorker(self.master_connection, self.request_queue) for _ in range(self.pool_size)] - - def terminate_pool(self): - if not self.terminated: - if log_enabled(BASIC): - log(BASIC, 'terminating pool <%s>', self) - self.started = False - self.request_queue.join() # waits for all queue pending operations - for _ in range(len([worker for worker in self.workers if worker.thread.is_alive()])): # put a TERMINATE signal on the queue for each active thread - self.request_queue.put((TERMINATE_REUSABLE, None, None, None)) - self.request_queue.join() # waits for all queue terminate operations - self.terminated = True - if log_enabled(BASIC): - log(BASIC, 'pool terminated for <%s>', self) - - class PooledConnectionThread(Thread): - """ - The thread that holds the Reusable connection and receive operation request via the queue - Result are sent back in the pool._incoming list when ready - """ - def __init__(self, worker, master_connection): - Thread.__init__(self) - self.daemon = True - self.worker = worker - self.master_connection = master_connection - if log_enabled(BASIC): - log(BASIC, 'instantiated PooledConnectionThread: <%r>', self) - - # noinspection PyProtectedMember - def run(self): - self.worker.running = True - terminate = False - pool = self.master_connection.strategy.pool - while not terminate: - try: - counter, message_type, request, controls = pool.request_queue.get(block=True, timeout=self.master_connection.strategy.pool.keepalive) - except Empty: # issue an Abandon(0) operation to keep the connection live - Abandon(0) is a harmless operation - if not self.worker.connection.closed: - self.worker.connection.abandon(0) - continue - - with self.worker.worker_lock: - self.worker.busy = True - if counter == TERMINATE_REUSABLE: - terminate = True - if self.worker.connection.bound: - try: - self.worker.connection.unbind() - if log_enabled(BASIC): - log(BASIC, 'thread terminated') - except LDAPExceptionError: - pass - else: - if (datetime.now() - self.worker.creation_time).seconds >= self.master_connection.strategy.pool.lifetime: # destroy and create a new connection - try: - self.worker.connection.unbind() - except LDAPExceptionError: - pass - self.worker.new_connection() - if log_enabled(BASIC): - log(BASIC, 'thread respawn') - if message_type not in ['bindRequest', 'unbindRequest']: - try: - if pool.open_pool and self.worker.connection.closed: - self.worker.connection.open(read_server_info=False) - if pool.tls_pool and not self.worker.connection.tls_started: - self.worker.connection.start_tls(read_server_info=False) - if pool.bind_pool and not self.worker.connection.bound: - self.worker.connection.bind(read_server_info=False) - elif pool.open_pool and not self.worker.connection.closed: # connection already open, issues a start_tls - if pool.tls_pool and not self.worker.connection.tls_started: - self.worker.connection.start_tls(read_server_info=False) - if self.worker.get_info_from_server and counter: - self.worker.connection._fire_deferred() - self.worker.get_info_from_server = False - response = None - result = None - if message_type == 'searchRequest': - response = self.worker.connection.post_send_search(self.worker.connection.send(message_type, request, controls)) - else: - response = self.worker.connection.post_send_single_response(self.worker.connection.send(message_type, request, controls)) - result = self.worker.connection.result - with pool.pool_lock: - pool._incoming[counter] = (response, result, BaseStrategy.decode_request(message_type, request, controls)) - except LDAPOperationResult as e: # raise_exceptions has raised an exception. It must be redirected to the original connection thread - with pool.pool_lock: - pool._incoming[counter] = (type(e)(str(e)), None, None) - # except LDAPOperationResult as e: # raise_exceptions has raised an exception. It must be redirected to the original connection thread - # exc = e - # with pool.pool_lock: - # if exc: - # pool._incoming[counter] = (exc, None, None) - # else: - # pool._incoming[counter] = (response, result, BaseStrategy.decode_request(message_type, request, controls)) - - self.worker.busy = False - pool.request_queue.task_done() - self.worker.task_counter += 1 - if log_enabled(BASIC): - log(BASIC, 'thread terminated') - if self.master_connection.usage: - pool.terminated_usage += self.worker.connection.usage - self.worker.running = False - - class PooledConnectionWorker(object): - """ - Container for the restartable connection. it includes a thread and a lock to execute the connection in the pool - """ - def __init__(self, connection, request_queue): - self.master_connection = connection - self.request_queue = request_queue - self.running = False - self.busy = False - self.get_info_from_server = False - self.connection = None - self.creation_time = None - self.task_counter = 0 - self.new_connection() - self.thread = ReusableStrategy.PooledConnectionThread(self, self.master_connection) - self.worker_lock = Lock() - if log_enabled(BASIC): - log(BASIC, 'instantiated PooledConnectionWorker: <%s>', self) - - def __str__(self): - s = 'CONN: ' + str(self.connection) + linesep + ' THREAD: ' - s += 'running' if self.running else 'halted' - s += ' - ' + ('busy' if self.busy else 'available') - s += ' - ' + ('created at: ' + self.creation_time.isoformat()) - s += ' - time to live: ' + str(self.master_connection.strategy.pool.lifetime - (datetime.now() - self.creation_time).seconds) - s += ' - requests served: ' + str(self.task_counter) - - return s - - def new_connection(self): - from ..core.connection import Connection - # noinspection PyProtectedMember - self.creation_time = datetime.now() - self.connection = Connection(server=self.master_connection.server_pool if self.master_connection.server_pool else self.master_connection.server, - user=self.master_connection.user, - password=self.master_connection.password, - auto_bind=AUTO_BIND_NONE, # do not perform auto_bind because it reads again the schema - version=self.master_connection.version, - authentication=self.master_connection.authentication, - client_strategy=RESTARTABLE, - auto_referrals=self.master_connection.auto_referrals, - auto_range=self.master_connection.auto_range, - sasl_mechanism=self.master_connection.sasl_mechanism, - sasl_credentials=self.master_connection.sasl_credentials, - check_names=self.master_connection.check_names, - collect_usage=self.master_connection._usage, - read_only=self.master_connection.read_only, - raise_exceptions=self.master_connection.raise_exceptions, - lazy=False, - fast_decoder=self.master_connection.fast_decoder, - receive_timeout=self.master_connection.receive_timeout, - return_empty_attributes=self.master_connection.empty_attributes) - - # simulates auto_bind, always with read_server_info=False - if self.master_connection.auto_bind and self.master_connection.auto_bind not in [AUTO_BIND_NONE, AUTO_BIND_DEFAULT]: - if log_enabled(BASIC): - log(BASIC, 'performing automatic bind for <%s>', self.connection) - self.connection.open(read_server_info=False) - if self.master_connection.auto_bind == AUTO_BIND_NO_TLS: - self.connection.bind(read_server_info=False) - elif self.master_connection.auto_bind == AUTO_BIND_TLS_BEFORE_BIND: - self.connection.start_tls(read_server_info=False) - self.connection.bind(read_server_info=False) - elif self.master_connection.auto_bind == AUTO_BIND_TLS_AFTER_BIND: - self.connection.bind(read_server_info=False) - self.connection.start_tls(read_server_info=False) - - if self.master_connection.server_pool: - self.connection.server_pool = self.master_connection.server_pool - self.connection.server_pool.initialize(self.connection) - - # ReusableStrategy methods - def __init__(self, ldap_connection): - BaseStrategy.__init__(self, ldap_connection) - self.sync = False - self.no_real_dsa = False - self.pooled = True - self.can_stream = False - if hasattr(ldap_connection, 'pool_name') and ldap_connection.pool_name: - self.pool = ReusableStrategy.ConnectionPool(ldap_connection) - else: - if log_enabled(ERROR): - log(ERROR, 'reusable connection must have a pool_name') - raise LDAPConnectionPoolNameIsMandatoryError('reusable connection must have a pool_name') - - def open(self, reset_usage=True, read_server_info=True): - # read_server_info not used - self.pool.open_pool = True - self.pool.start_pool() - self.connection.closed = False - if self.connection.usage: - if reset_usage or not self.connection._usage.initial_connection_start_time: - self.connection._usage.start() - - def terminate(self): - self.pool.terminate_pool() - self.pool.open_pool = False - self.connection.bound = False - self.connection.closed = True - self.pool.bind_pool = False - self.pool.tls_pool = False - - def _close_socket(self): - """ - Doesn't really close the socket - """ - self.connection.closed = True - - if self.connection.usage: - self.connection._usage.closed_sockets += 1 - - def send(self, message_type, request, controls=None): - if self.pool.started: - if message_type == 'bindRequest': - self.pool.bind_pool = True - counter = BOGUS_BIND - elif message_type == 'unbindRequest': - self.pool.bind_pool = False - counter = BOGUS_UNBIND - elif message_type == 'abandonRequest': - counter = BOGUS_ABANDON - elif message_type == 'extendedReq' and self.connection.starting_tls: - self.pool.tls_pool = True - counter = BOGUS_EXTENDED - else: - with self.pool.pool_lock: - self.pool.counter += 1 - if self.pool.counter > LDAP_MAX_INT: - self.pool.counter = 1 - counter = self.pool.counter - self.pool.request_queue.put((counter, message_type, request, controls)) - return counter - if log_enabled(ERROR): - log(ERROR, 'reusable connection pool not started') - raise LDAPConnectionPoolNotStartedError('reusable connection pool not started') - - def validate_bind(self, controls): - # in case of a new connection or different credentials - if (self.connection.user != self.pool.master_connection.user or - self.connection.password != self.pool.master_connection.password or - self.connection.authentication != self.pool.master_connection.authentication or - self.connection.sasl_mechanism != self.pool.master_connection.sasl_mechanism or - self.connection.sasl_credentials != self.pool.master_connection.sasl_credentials): - self.pool.master_connection.user = self.connection.user - self.pool.master_connection.password = self.connection.password - self.pool.master_connection.authentication = self.connection.authentication - self.pool.master_connection.sasl_mechanism = self.connection.sasl_mechanism - self.pool.master_connection.sasl_credentials = self.connection.sasl_credentials - self.pool.rebind_pool() - temp_connection = self.pool.workers[0].connection - temp_connection.lazy = False - if not self.connection.server.schema or not self.connection.server.info: - result = self.pool.workers[0].connection.bind(controls=controls) - else: - result = self.pool.workers[0].connection.bind(controls=controls, read_server_info=False) - - temp_connection.unbind() - temp_connection.lazy = True - if result: - self.pool.bind_pool = True # bind pool if bind is validated - return result - - def get_response(self, counter, timeout=None, get_request=False): - sleeptime = get_config_parameter('RESPONSE_SLEEPTIME') - request=None - if timeout is None: - timeout = get_config_parameter('RESPONSE_WAITING_TIMEOUT') - if counter == BOGUS_BIND: # send a bogus bindResponse - response = list() - result = {'description': 'success', 'referrals': None, 'type': 'bindResponse', 'result': 0, 'dn': '', 'message': '', 'saslCreds': None} - elif counter == BOGUS_UNBIND: # bogus unbind response - response = None - result = None - elif counter == BOGUS_ABANDON: # abandon cannot be executed because of multiple connections - response = list() - result = {'result': 0, 'referrals': None, 'responseName': '1.3.6.1.4.1.1466.20037', 'type': 'extendedResp', 'description': 'success', 'responseValue': 'None', 'dn': '', 'message': ''} - elif counter == BOGUS_EXTENDED: # bogus startTls extended response - response = list() - result = {'result': 0, 'referrals': None, 'responseName': '1.3.6.1.4.1.1466.20037', 'type': 'extendedResp', 'description': 'success', 'responseValue': 'None', 'dn': '', 'message': ''} - self.connection.starting_tls = False - else: - response = None - result = None - while timeout >= 0: # waiting for completed message to appear in _incoming - try: - with self.connection.strategy.pool.pool_lock: - response, result, request = self.connection.strategy.pool._incoming.pop(counter) - except KeyError: - sleep(sleeptime) - timeout -= sleeptime - continue - break - - if timeout <= 0: - if log_enabled(ERROR): - log(ERROR, 'no response from worker threads in Reusable connection') - raise LDAPResponseTimeoutError('no response from worker threads in Reusable connection') - - if isinstance(response, LDAPOperationResult): - raise response # an exception has been raised with raise_exceptions - - if get_request: - return response, result, request - - return response, result - - def post_send_single_response(self, counter): - return counter - - def post_send_search(self, counter): - return counter +""" +""" + +# Created on 2014.03.23 +# +# Author: Giovanni Cannata +# +# Copyright 2014 - 2019 Giovanni Cannata +# +# This file is part of ldap3. +# +# ldap3 is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ldap3 is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with ldap3 in the COPYING and COPYING.LESSER files. +# If not, see . + +from datetime import datetime +from os import linesep +from threading import Thread, Lock +from time import sleep + +from .. import RESTARTABLE, get_config_parameter, AUTO_BIND_DEFAULT, AUTO_BIND_NONE, AUTO_BIND_NO_TLS, AUTO_BIND_TLS_AFTER_BIND, AUTO_BIND_TLS_BEFORE_BIND +from .base import BaseStrategy +from ..core.usage import ConnectionUsage +from ..core.exceptions import LDAPConnectionPoolNameIsMandatoryError, LDAPConnectionPoolNotStartedError, LDAPOperationResult, LDAPExceptionError, LDAPResponseTimeoutError +from ..utils.log import log, log_enabled, ERROR, BASIC +from ..protocol.rfc4511 import LDAP_MAX_INT + +TERMINATE_REUSABLE = 'TERMINATE_REUSABLE_CONNECTION' + +BOGUS_BIND = -1 +BOGUS_UNBIND = -2 +BOGUS_EXTENDED = -3 +BOGUS_ABANDON = -4 + +try: + from queue import Queue, Empty +except ImportError: # Python 2 + # noinspection PyUnresolvedReferences + from Queue import Queue, Empty + + +# noinspection PyProtectedMember +class ReusableStrategy(BaseStrategy): + """ + A pool of reusable SyncWaitRestartable connections with lazy behaviour and limited lifetime. + The connection using this strategy presents itself as a normal connection, but internally the strategy has a pool of + connections that can be used as needed. Each connection lives in its own thread and has a busy/available status. + The strategy performs the requested operation on the first available connection. + The pool of connections is instantiated at strategy initialization. + Strategy has two customizable properties, the total number of connections in the pool and the lifetime of each connection. + When lifetime is expired the connection is closed and will be open again when needed. + """ + pools = dict() + + def receiving(self): + raise NotImplementedError + + def _start_listen(self): + raise NotImplementedError + + def _get_response(self, message_id): + raise NotImplementedError + + def get_stream(self): + raise NotImplementedError + + def set_stream(self, value): + raise NotImplementedError + + # noinspection PyProtectedMember + class ConnectionPool(object): + """ + Container for the Connection Threads + """ + def __new__(cls, connection): + if connection.pool_name in ReusableStrategy.pools: # returns existing connection pool + pool = ReusableStrategy.pools[connection.pool_name] + if not pool.started: # if pool is not started remove it from the pools singleton and create a new onw + del ReusableStrategy.pools[connection.pool_name] + return object.__new__(cls) + if connection.pool_keepalive and pool.keepalive != connection.pool_keepalive: # change lifetime + pool.keepalive = connection.pool_keepalive + if connection.pool_lifetime and pool.lifetime != connection.pool_lifetime: # change keepalive + pool.lifetime = connection.pool_lifetime + if connection.pool_size and pool.pool_size != connection.pool_size: # if pool size has changed terminate and recreate the connections + pool.terminate_pool() + pool.pool_size = connection.pool_size + return pool + else: + return object.__new__(cls) + + def __init__(self, connection): + if not hasattr(self, 'workers'): + self.name = connection.pool_name + self.master_connection = connection + self.workers = [] + self.pool_size = connection.pool_size or get_config_parameter('REUSABLE_THREADED_POOL_SIZE') + self.lifetime = connection.pool_lifetime or get_config_parameter('REUSABLE_THREADED_LIFETIME') + self.keepalive = connection.pool_keepalive + self.request_queue = Queue() + self.open_pool = False + self.bind_pool = False + self.tls_pool = False + self._incoming = dict() + self.counter = 0 + self.terminated_usage = ConnectionUsage() if connection._usage else None + self.terminated = False + self.pool_lock = Lock() + ReusableStrategy.pools[self.name] = self + self.started = False + if log_enabled(BASIC): + log(BASIC, 'instantiated ConnectionPool: <%r>', self) + + def __str__(self): + s = 'POOL: ' + str(self.name) + ' - status: ' + ('started' if self.started else 'terminated') + s += ' - responses in queue: ' + str(len(self._incoming)) + s += ' - pool size: ' + str(self.pool_size) + s += ' - lifetime: ' + str(self.lifetime) + s += ' - keepalive: ' + str(self.keepalive) + s += ' - open: ' + str(self.open_pool) + s += ' - bind: ' + str(self.bind_pool) + s += ' - tls: ' + str(self.tls_pool) + linesep + s += 'MASTER CONN: ' + str(self.master_connection) + linesep + s += 'WORKERS:' + if self.workers: + for i, worker in enumerate(self.workers): + s += linesep + str(i).rjust(5) + ': ' + str(worker) + else: + s += linesep + ' no active workers in pool' + + return s + + def __repr__(self): + return self.__str__() + + def get_info_from_server(self): + for worker in self.workers: + with worker.worker_lock: + if not worker.connection.server.schema or not worker.connection.server.info: + worker.get_info_from_server = True + else: + worker.get_info_from_server = False + + def rebind_pool(self): + for worker in self.workers: + with worker.worker_lock: + worker.connection.rebind(self.master_connection.user, + self.master_connection.password, + self.master_connection.authentication, + self.master_connection.sasl_mechanism, + self.master_connection.sasl_credentials) + + def start_pool(self): + if not self.started: + self.create_pool() + for worker in self.workers: + with worker.worker_lock: + worker.thread.start() + self.started = True + self.terminated = False + if log_enabled(BASIC): + log(BASIC, 'worker started for pool <%s>', self) + return True + return False + + def create_pool(self): + if log_enabled(BASIC): + log(BASIC, 'created pool <%s>', self) + self.workers = [ReusableStrategy.PooledConnectionWorker(self.master_connection, self.request_queue) for _ in range(self.pool_size)] + + def terminate_pool(self): + if not self.terminated: + if log_enabled(BASIC): + log(BASIC, 'terminating pool <%s>', self) + self.started = False + self.request_queue.join() # waits for all queue pending operations + for _ in range(len([worker for worker in self.workers if worker.thread.is_alive()])): # put a TERMINATE signal on the queue for each active thread + self.request_queue.put((TERMINATE_REUSABLE, None, None, None)) + self.request_queue.join() # waits for all queue terminate operations + self.terminated = True + if log_enabled(BASIC): + log(BASIC, 'pool terminated for <%s>', self) + + class PooledConnectionThread(Thread): + """ + The thread that holds the Reusable connection and receive operation request via the queue + Result are sent back in the pool._incoming list when ready + """ + def __init__(self, worker, master_connection): + Thread.__init__(self) + self.daemon = True + self.worker = worker + self.master_connection = master_connection + if log_enabled(BASIC): + log(BASIC, 'instantiated PooledConnectionThread: <%r>', self) + + # noinspection PyProtectedMember + def run(self): + self.worker.running = True + terminate = False + pool = self.master_connection.strategy.pool + while not terminate: + try: + counter, message_type, request, controls = pool.request_queue.get(block=True, timeout=self.master_connection.strategy.pool.keepalive) + except Empty: # issue an Abandon(0) operation to keep the connection live - Abandon(0) is a harmless operation + if not self.worker.connection.closed: + self.worker.connection.abandon(0) + continue + + with self.worker.worker_lock: + self.worker.busy = True + if counter == TERMINATE_REUSABLE: + terminate = True + if self.worker.connection.bound: + try: + self.worker.connection.unbind() + if log_enabled(BASIC): + log(BASIC, 'thread terminated') + except LDAPExceptionError: + pass + else: + if (datetime.now() - self.worker.creation_time).seconds >= self.master_connection.strategy.pool.lifetime: # destroy and create a new connection + try: + self.worker.connection.unbind() + except LDAPExceptionError: + pass + self.worker.new_connection() + if log_enabled(BASIC): + log(BASIC, 'thread respawn') + if message_type not in ['bindRequest', 'unbindRequest']: + try: + if pool.open_pool and self.worker.connection.closed: + self.worker.connection.open(read_server_info=False) + if pool.tls_pool and not self.worker.connection.tls_started: + self.worker.connection.start_tls(read_server_info=False) + if pool.bind_pool and not self.worker.connection.bound: + self.worker.connection.bind(read_server_info=False) + elif pool.open_pool and not self.worker.connection.closed: # connection already open, issues a start_tls + if pool.tls_pool and not self.worker.connection.tls_started: + self.worker.connection.start_tls(read_server_info=False) + if self.worker.get_info_from_server and counter: + self.worker.connection.refresh_server_info() + self.worker.get_info_from_server = False + response = None + result = None + if message_type == 'searchRequest': + response = self.worker.connection.post_send_search(self.worker.connection.send(message_type, request, controls)) + else: + response = self.worker.connection.post_send_single_response(self.worker.connection.send(message_type, request, controls)) + result = self.worker.connection.result + with pool.pool_lock: + pool._incoming[counter] = (response, result, BaseStrategy.decode_request(message_type, request, controls)) + except LDAPOperationResult as e: # raise_exceptions has raised an exception. It must be redirected to the original connection thread + with pool.pool_lock: + pool._incoming[counter] = (e, None, None) + # pool._incoming[counter] = (type(e)(str(e)), None, None) + # except LDAPOperationResult as e: # raise_exceptions has raised an exception. It must be redirected to the original connection thread + # exc = e + # with pool.pool_lock: + # if exc: + # pool._incoming[counter] = (exc, None, None) + # else: + # pool._incoming[counter] = (response, result, BaseStrategy.decode_request(message_type, request, controls)) + + self.worker.busy = False + pool.request_queue.task_done() + self.worker.task_counter += 1 + if log_enabled(BASIC): + log(BASIC, 'thread terminated') + if self.master_connection.usage: + pool.terminated_usage += self.worker.connection.usage + self.worker.running = False + + class PooledConnectionWorker(object): + """ + Container for the restartable connection. it includes a thread and a lock to execute the connection in the pool + """ + def __init__(self, connection, request_queue): + self.master_connection = connection + self.request_queue = request_queue + self.running = False + self.busy = False + self.get_info_from_server = False + self.connection = None + self.creation_time = None + self.task_counter = 0 + self.new_connection() + self.thread = ReusableStrategy.PooledConnectionThread(self, self.master_connection) + self.worker_lock = Lock() + if log_enabled(BASIC): + log(BASIC, 'instantiated PooledConnectionWorker: <%s>', self) + + def __str__(self): + s = 'CONN: ' + str(self.connection) + linesep + ' THREAD: ' + s += 'running' if self.running else 'halted' + s += ' - ' + ('busy' if self.busy else 'available') + s += ' - ' + ('created at: ' + self.creation_time.isoformat()) + s += ' - time to live: ' + str(self.master_connection.strategy.pool.lifetime - (datetime.now() - self.creation_time).seconds) + s += ' - requests served: ' + str(self.task_counter) + + return s + + def new_connection(self): + from ..core.connection import Connection + # noinspection PyProtectedMember + self.creation_time = datetime.now() + self.connection = Connection(server=self.master_connection.server_pool if self.master_connection.server_pool else self.master_connection.server, + user=self.master_connection.user, + password=self.master_connection.password, + auto_bind=AUTO_BIND_NONE, # do not perform auto_bind because it reads again the schema + version=self.master_connection.version, + authentication=self.master_connection.authentication, + client_strategy=RESTARTABLE, + auto_referrals=self.master_connection.auto_referrals, + auto_range=self.master_connection.auto_range, + sasl_mechanism=self.master_connection.sasl_mechanism, + sasl_credentials=self.master_connection.sasl_credentials, + check_names=self.master_connection.check_names, + collect_usage=self.master_connection._usage, + read_only=self.master_connection.read_only, + raise_exceptions=self.master_connection.raise_exceptions, + lazy=False, + fast_decoder=self.master_connection.fast_decoder, + receive_timeout=self.master_connection.receive_timeout, + return_empty_attributes=self.master_connection.empty_attributes) + + # simulates auto_bind, always with read_server_info=False + if self.master_connection.auto_bind and self.master_connection.auto_bind not in [AUTO_BIND_NONE, AUTO_BIND_DEFAULT]: + if log_enabled(BASIC): + log(BASIC, 'performing automatic bind for <%s>', self.connection) + self.connection.open(read_server_info=False) + if self.master_connection.auto_bind == AUTO_BIND_NO_TLS: + self.connection.bind(read_server_info=False) + elif self.master_connection.auto_bind == AUTO_BIND_TLS_BEFORE_BIND: + self.connection.start_tls(read_server_info=False) + self.connection.bind(read_server_info=False) + elif self.master_connection.auto_bind == AUTO_BIND_TLS_AFTER_BIND: + self.connection.bind(read_server_info=False) + self.connection.start_tls(read_server_info=False) + + if self.master_connection.server_pool: + self.connection.server_pool = self.master_connection.server_pool + self.connection.server_pool.initialize(self.connection) + + # ReusableStrategy methods + def __init__(self, ldap_connection): + BaseStrategy.__init__(self, ldap_connection) + self.sync = False + self.no_real_dsa = False + self.pooled = True + self.can_stream = False + if hasattr(ldap_connection, 'pool_name') and ldap_connection.pool_name: + self.pool = ReusableStrategy.ConnectionPool(ldap_connection) + else: + if log_enabled(ERROR): + log(ERROR, 'reusable connection must have a pool_name') + raise LDAPConnectionPoolNameIsMandatoryError('reusable connection must have a pool_name') + + def open(self, reset_usage=True, read_server_info=True): + # read_server_info not used + self.pool.open_pool = True + self.pool.start_pool() + self.connection.closed = False + if self.connection.usage: + if reset_usage or not self.connection._usage.initial_connection_start_time: + self.connection._usage.start() + + def terminate(self): + self.pool.terminate_pool() + self.pool.open_pool = False + self.connection.bound = False + self.connection.closed = True + self.pool.bind_pool = False + self.pool.tls_pool = False + + def _close_socket(self): + """ + Doesn't really close the socket + """ + self.connection.closed = True + + if self.connection.usage: + self.connection._usage.closed_sockets += 1 + + def send(self, message_type, request, controls=None): + if self.pool.started: + if message_type == 'bindRequest': + self.pool.bind_pool = True + counter = BOGUS_BIND + elif message_type == 'unbindRequest': + self.pool.bind_pool = False + counter = BOGUS_UNBIND + elif message_type == 'abandonRequest': + counter = BOGUS_ABANDON + elif message_type == 'extendedReq' and self.connection.starting_tls: + self.pool.tls_pool = True + counter = BOGUS_EXTENDED + else: + with self.pool.pool_lock: + self.pool.counter += 1 + if self.pool.counter > LDAP_MAX_INT: + self.pool.counter = 1 + counter = self.pool.counter + self.pool.request_queue.put((counter, message_type, request, controls)) + return counter + if log_enabled(ERROR): + log(ERROR, 'reusable connection pool not started') + raise LDAPConnectionPoolNotStartedError('reusable connection pool not started') + + def validate_bind(self, controls): + # in case of a new connection or different credentials + if (self.connection.user != self.pool.master_connection.user or + self.connection.password != self.pool.master_connection.password or + self.connection.authentication != self.pool.master_connection.authentication or + self.connection.sasl_mechanism != self.pool.master_connection.sasl_mechanism or + self.connection.sasl_credentials != self.pool.master_connection.sasl_credentials): + self.pool.master_connection.user = self.connection.user + self.pool.master_connection.password = self.connection.password + self.pool.master_connection.authentication = self.connection.authentication + self.pool.master_connection.sasl_mechanism = self.connection.sasl_mechanism + self.pool.master_connection.sasl_credentials = self.connection.sasl_credentials + self.pool.rebind_pool() + temp_connection = self.pool.workers[0].connection + old_lazy = temp_connection.lazy + temp_connection.lazy = False + if not self.connection.server.schema or not self.connection.server.info: + result = self.pool.workers[0].connection.bind(controls=controls) + else: + result = self.pool.workers[0].connection.bind(controls=controls, read_server_info=False) + + temp_connection.unbind() + temp_connection.lazy = old_lazy + if result: + self.pool.bind_pool = True # bind pool if bind is validated + return result + + def get_response(self, counter, timeout=None, get_request=False): + sleeptime = get_config_parameter('RESPONSE_SLEEPTIME') + request=None + if timeout is None: + timeout = get_config_parameter('RESPONSE_WAITING_TIMEOUT') + if counter == BOGUS_BIND: # send a bogus bindResponse + response = list() + result = {'description': 'success', 'referrals': None, 'type': 'bindResponse', 'result': 0, 'dn': '', 'message': '', 'saslCreds': None} + elif counter == BOGUS_UNBIND: # bogus unbind response + response = None + result = None + elif counter == BOGUS_ABANDON: # abandon cannot be executed because of multiple connections + response = list() + result = {'result': 0, 'referrals': None, 'responseName': '1.3.6.1.4.1.1466.20037', 'type': 'extendedResp', 'description': 'success', 'responseValue': 'None', 'dn': '', 'message': ''} + elif counter == BOGUS_EXTENDED: # bogus startTls extended response + response = list() + result = {'result': 0, 'referrals': None, 'responseName': '1.3.6.1.4.1.1466.20037', 'type': 'extendedResp', 'description': 'success', 'responseValue': 'None', 'dn': '', 'message': ''} + self.connection.starting_tls = False + else: + response = None + result = None + while timeout >= 0: # waiting for completed message to appear in _incoming + try: + with self.connection.strategy.pool.pool_lock: + response, result, request = self.connection.strategy.pool._incoming.pop(counter) + except KeyError: + sleep(sleeptime) + timeout -= sleeptime + continue + break + + if timeout <= 0: + if log_enabled(ERROR): + log(ERROR, 'no response from worker threads in Reusable connection') + raise LDAPResponseTimeoutError('no response from worker threads in Reusable connection') + + if isinstance(response, LDAPOperationResult): + raise response # an exception has been raised with raise_exceptions + + if get_request: + return response, result, request + + return response, result + + def post_send_single_response(self, counter): + return counter + + def post_send_search(self, counter): + return counter diff --git a/server/www/packages/packages-linux/x64/ldap3/strategy/sync.py b/server/www/packages/packages-linux/x64/ldap3/strategy/sync.py index b2c0257..fc7ccfb 100644 --- a/server/www/packages/packages-linux/x64/ldap3/strategy/sync.py +++ b/server/www/packages/packages-linux/x64/ldap3/strategy/sync.py @@ -67,7 +67,7 @@ class SyncStrategy(BaseStrategy): def receiving(self): """ - Receive data over the socket + Receives data over the socket Checks if the socket is closed """ messages = [] diff --git a/server/www/packages/packages-linux/x64/ldap3/utils/ciDict.py b/server/www/packages/packages-linux/x64/ldap3/utils/ciDict.py index f81ba1b..baf5d38 100644 --- a/server/www/packages/packages-linux/x64/ldap3/utils/ciDict.py +++ b/server/www/packages/packages-linux/x64/ldap3/utils/ciDict.py @@ -5,7 +5,7 @@ # # Author: Giovanni Cannata # -# Copyright 2014 - 2018 Giovanni Cannata +# Copyright 2014 - 2019 Giovanni Cannata # # This file is part of ldap3. # @@ -143,7 +143,7 @@ class CaseInsensitiveWithAliasDict(CaseInsensitiveDict): if ci_key in self._aliases: self.remove_alias(ci_key) - def set_alias(self, key, alias): + def set_alias(self, key, alias, ignore_duplicates=False): if not isinstance(alias, SEQUENCE_TYPES): alias = [alias] for alias_to_add in alias: @@ -153,20 +153,20 @@ class CaseInsensitiveWithAliasDict(CaseInsensitiveDict): if ci_alias not in self._case_insensitive_keymap: # checks if alias is used a key if ci_alias not in self._aliases: # checks if alias is used as another alias self._aliases[ci_alias] = ci_key - if ci_key in self._alias_keymap: # extend alias keymap + if ci_key in self._alias_keymap: # extends alias keymap self._alias_keymap[ci_key].append(self._ci_key(ci_alias)) else: self._alias_keymap[ci_key] = list() self._alias_keymap[ci_key].append(self._ci_key(ci_alias)) else: - if ci_key == self._ci_key(self._alias_keymap[ci_alias]): # passes if alias is already defined to the same key + if ci_key in self._alias_keymap and ci_alias in self._alias_keymap[ci_key]: # passes if alias is already defined to the same key pass - else: + elif not ignore_duplicates: raise KeyError('\'' + str(alias_to_add) + '\' already used as alias') else: if ci_key == self._ci_key(self._case_insensitive_keymap[ci_alias]): # passes if alias is already defined to the same key pass - else: + elif not ignore_duplicates: raise KeyError('\'' + str(alias_to_add) + '\' already used as key') else: raise KeyError('\'' + str(ci_key) + '\' is not an existing key') diff --git a/server/www/packages/packages-linux/x64/ldap3/utils/conv.py b/server/www/packages/packages-linux/x64/ldap3/utils/conv.py index ee90c66..9970926 100644 --- a/server/www/packages/packages-linux/x64/ldap3/utils/conv.py +++ b/server/www/packages/packages-linux/x64/ldap3/utils/conv.py @@ -1,278 +1,270 @@ -""" -""" - -# Created on 2014.04.26 -# -# Author: Giovanni Cannata -# -# Copyright 2014 - 2018 Giovanni Cannata -# -# This file is part of ldap3. -# -# ldap3 is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# ldap3 is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with ldap3 in the COPYING and COPYING.LESSER files. -# If not, see . - -from base64 import b64encode, b64decode -import datetime -import re - -from .. import SEQUENCE_TYPES, STRING_TYPES, NUMERIC_TYPES, get_config_parameter -from ..utils.ciDict import CaseInsensitiveDict -from ..core.exceptions import LDAPDefinitionError - - -def to_unicode(obj, encoding=None, from_server=False): - """Try to convert bytes (and str in python2) to unicode. - Return object unmodified if python3 string, else raise an exception - """ - conf_default_client_encoding = get_config_parameter('DEFAULT_CLIENT_ENCODING') - conf_default_server_encoding = get_config_parameter('DEFAULT_SERVER_ENCODING') - conf_additional_server_encodings = get_config_parameter('ADDITIONAL_SERVER_ENCODINGS') - conf_additional_client_encodings = get_config_parameter('ADDITIONAL_CLIENT_ENCODINGS') - if isinstance(obj, NUMERIC_TYPES): - obj = str(obj) - - if isinstance(obj, (bytes, bytearray)): - if from_server: # data from server - if encoding is None: - encoding = conf_default_server_encoding - try: - return obj.decode(encoding) - except UnicodeDecodeError: - for encoding in conf_additional_server_encodings: # AD could have DN not encoded in utf-8 (even if this is not allowed by RFC4510) - try: - return obj.decode(encoding) - except UnicodeDecodeError: - pass - raise UnicodeError("Unable to convert server data to unicode: %r" % obj) - else: # data from client - if encoding is None: - encoding = conf_default_client_encoding - try: - return obj.decode(encoding) - except UnicodeDecodeError: - for encoding in conf_additional_client_encodings: # tries additional encodings - try: - return obj.decode(encoding) - except UnicodeDecodeError: - pass - raise UnicodeError("Unable to convert client data to unicode: %r" % obj) - - if isinstance(obj, STRING_TYPES): # python3 strings, python 2 unicode - return obj - - raise UnicodeError("Unable to convert type %s to unicode: %r" % (type(obj).__class__.__name__, obj)) - - -def to_raw(obj, encoding='utf-8'): - """Tries to convert to raw bytes from unicode""" - if isinstance(obj, NUMERIC_TYPES): - obj = str(obj) - - if not (isinstance(obj, bytes)): - if isinstance(obj, SEQUENCE_TYPES): - return [to_raw(element) for element in obj] - elif isinstance(obj, STRING_TYPES): - return obj.encode(encoding) - return obj - - -def escape_filter_chars(text, encoding=None): - """ Escape chars mentioned in RFC4515. """ - if encoding is None: - encoding = get_config_parameter('DEFAULT_ENCODING') - - try: - text = to_unicode(text, encoding) - escaped = text.replace('\\', '\\5c') - escaped = escaped.replace('*', '\\2a') - escaped = escaped.replace('(', '\\28') - escaped = escaped.replace(')', '\\29') - escaped = escaped.replace('\x00', '\\00') - except Exception: # probably raw bytes values, return escaped bytes value - escaped = to_unicode(escape_bytes(text)) - # escape all octets greater than 0x7F that are not part of a valid UTF-8 - # escaped = ''.join(c if c <= ord(b'\x7f') else escape_bytes(to_raw(to_unicode(c, encoding))) for c in escaped) - return escaped - - -def unescape_filter_chars(text, encoding=None): - """ unescape chars mentioned in RFC4515. """ - if encoding is None: - encoding = get_config_parameter('DEFAULT_ENCODING') - - unescaped = to_raw(text, encoding) - unescaped = unescaped.replace(b'\\5c', b'\\') - unescaped = unescaped.replace(b'\\5C', b'\\') - unescaped = unescaped.replace(b'\\2a', b'*') - unescaped = unescaped.replace(b'\\2A', b'*') - unescaped = unescaped.replace(b'\\28', b'(') - unescaped = unescaped.replace(b'\\29', b')') - unescaped = unescaped.replace(b'\\00', b'\x00') - return unescaped - - -def escape_bytes(bytes_value): - """ Convert a byte sequence to a properly escaped for LDAP (format BACKSLASH HEX HEX) string""" - if bytes_value: - if str is not bytes: # Python 3 - if isinstance(bytes_value, str): - bytes_value = bytearray(bytes_value, encoding='utf-8') - escaped = '\\'.join([('%02x' % int(b)) for b in bytes_value]) - else: # Python 2 - if isinstance(bytes_value, unicode): - bytes_value = bytes_value.encode('utf-8') - escaped = '\\'.join([('%02x' % ord(b)) for b in bytes_value]) - else: - escaped = '' - - return ('\\' + escaped) if escaped else '' - - -def prepare_for_stream(value): - if str is not bytes: # Python 3 - return value - else: # Python 2 - return value.decode() - -def json_encode_b64(obj): - try: - return dict(encoding='base64', encoded=b64encode(obj)) - except Exception as e: - raise LDAPDefinitionError('unable to encode ' + str(obj) + ' - ' + str(e)) - - -# noinspection PyProtectedMember -def check_json_dict(json_dict): - # needed for python 2 - - for k, v in json_dict.items(): - if isinstance(v, dict): - check_json_dict(v) - elif isinstance(v, CaseInsensitiveDict): - check_json_dict(v._store) - elif isinstance(v, SEQUENCE_TYPES): - for i, e in enumerate(v): - if isinstance(e, dict): - check_json_dict(e) - elif isinstance(e, CaseInsensitiveDict): - check_json_dict(e._store) - else: - v[i] = format_json(e) - else: - json_dict[k] = format_json(v) - - -def json_hook(obj): - if hasattr(obj, 'keys') and len(list(obj.keys())) == 2 and 'encoding' in obj.keys() and 'encoded' in obj.keys(): - return b64decode(obj['encoded']) - - return obj - - -# noinspection PyProtectedMember -def format_json(obj): - if isinstance(obj, CaseInsensitiveDict): - return obj._store - - if isinstance(obj, datetime.datetime): - return str(obj) - - if isinstance(obj, int): - return obj - - if str is bytes: # Python 2 - if isinstance(obj, long): # long exists only in python2 - return obj - - try: - if str is not bytes: # Python 3 - if isinstance(obj, bytes): - # return check_escape(str(obj, 'utf-8', errors='strict')) - return str(obj, 'utf-8', errors='strict') - raise LDAPDefinitionError('unable to serialize ' + str(obj)) - else: # Python 2 - if isinstance(obj, unicode): - return obj - else: - # return unicode(check_escape(obj)) - return unicode(obj) - except (TypeError, UnicodeDecodeError): - pass - - try: - return json_encode_b64(bytes(obj)) - except Exception: - pass - - raise LDAPDefinitionError('unable to serialize ' + str(obj)) - - -def is_filter_escaped(text): - if not type(text) == ((str is not bytes) and str or unicode): # requires str for Python 3 and unicode for Python 2 - raise ValueError('unicode input expected') - - return all(c not in text for c in '()*\0') and not re.search('\\\\([^0-9a-fA-F]|(.[^0-9a-fA-F]))', text) - - -# def ldap_escape_to_bytes(text): -# bytesequence = bytearray() -# if text.startswith('\\'): -# byte_values = text.split('\\') -# for value in byte_values[1:]: -# if len(value) != 2 and not value.isdigit(): -# raise LDAPDefinitionError('badly formatted LDAP byte escaped sequence') -# bytesequence.append(int(value, 16)) -# return bytes(bytesequence) -# raise LDAPDefinitionError('badly formatted LDAP byte escaped sequence') - - -def ldap_escape_to_bytes(text): - bytesequence = bytearray() - i = 0 - try: - if isinstance(text, STRING_TYPES): - while i < len(text): - if text[i] == '\\': - if len(text) > i + 2: - try: - bytesequence.append(int(text[i+1:i+3], 16)) - i += 3 - continue - except ValueError: - pass - bytesequence.append(92) # "\" ASCII code - else: - raw = to_raw(text[i]) - for c in raw: - bytesequence.append(c) - i += 1 - elif isinstance(text, (bytes, bytearray)): - while i < len(text): - if text[i] == 92: # "\" ASCII code - if len(text) > i + 2: - try: - bytesequence.append(int(text[i + 1:i + 3], 16)) - i += 3 - continue - except ValueError: - pass - bytesequence.append(92) # "\" ASCII code - else: - bytesequence.append(text[i]) - i += 1 - except Exception: - raise LDAPDefinitionError('badly formatted LDAP byte escaped sequence') - - return bytes(bytesequence) +""" +""" + +# Created on 2014.04.26 +# +# Author: Giovanni Cannata +# +# Copyright 2014 - 2019 Giovanni Cannata +# +# This file is part of ldap3. +# +# ldap3 is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ldap3 is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with ldap3 in the COPYING and COPYING.LESSER files. +# If not, see . + +from base64 import b64encode, b64decode +import datetime +import re + +from .. import SEQUENCE_TYPES, STRING_TYPES, NUMERIC_TYPES, get_config_parameter +from ..utils.ciDict import CaseInsensitiveDict +from ..core.exceptions import LDAPDefinitionError + + +def to_unicode(obj, encoding=None, from_server=False): + """Try to convert bytes (and str in python2) to unicode. + Return object unmodified if python3 string, else raise an exception + """ + conf_default_client_encoding = get_config_parameter('DEFAULT_CLIENT_ENCODING') + conf_default_server_encoding = get_config_parameter('DEFAULT_SERVER_ENCODING') + conf_additional_server_encodings = get_config_parameter('ADDITIONAL_SERVER_ENCODINGS') + conf_additional_client_encodings = get_config_parameter('ADDITIONAL_CLIENT_ENCODINGS') + if isinstance(obj, NUMERIC_TYPES): + obj = str(obj) + + if isinstance(obj, (bytes, bytearray)): + if from_server: # data from server + if encoding is None: + encoding = conf_default_server_encoding + try: + return obj.decode(encoding) + except UnicodeDecodeError: + for encoding in conf_additional_server_encodings: # AD could have DN not encoded in utf-8 (even if this is not allowed by RFC4510) + try: + return obj.decode(encoding) + except UnicodeDecodeError: + pass + raise UnicodeError("Unable to convert server data to unicode: %r" % obj) + else: # data from client + if encoding is None: + encoding = conf_default_client_encoding + try: + return obj.decode(encoding) + except UnicodeDecodeError: + for encoding in conf_additional_client_encodings: # tries additional encodings + try: + return obj.decode(encoding) + except UnicodeDecodeError: + pass + raise UnicodeError("Unable to convert client data to unicode: %r" % obj) + + if isinstance(obj, STRING_TYPES): # python3 strings, python 2 unicode + return obj + + raise UnicodeError("Unable to convert type %s to unicode: %r" % (type(obj).__class__.__name__, obj)) + + +def to_raw(obj, encoding='utf-8'): + """Tries to convert to raw bytes from unicode""" + if isinstance(obj, NUMERIC_TYPES): + obj = str(obj) + + if not (isinstance(obj, bytes)): + if isinstance(obj, SEQUENCE_TYPES): + return [to_raw(element) for element in obj] + elif isinstance(obj, STRING_TYPES): + return obj.encode(encoding) + return obj + + +def escape_filter_chars(text, encoding=None): + """ Escape chars mentioned in RFC4515. """ + if encoding is None: + encoding = get_config_parameter('DEFAULT_ENCODING') + + try: + text = to_unicode(text, encoding) + escaped = text.replace('\\', '\\5c') + escaped = escaped.replace('*', '\\2a') + escaped = escaped.replace('(', '\\28') + escaped = escaped.replace(')', '\\29') + escaped = escaped.replace('\x00', '\\00') + except Exception: # probably raw bytes values, return escaped bytes value + escaped = to_unicode(escape_bytes(text)) + # escape all octets greater than 0x7F that are not part of a valid UTF-8 + # escaped = ''.join(c if c <= ord(b'\x7f') else escape_bytes(to_raw(to_unicode(c, encoding))) for c in escaped) + return escaped + + +def unescape_filter_chars(text, encoding=None): + """ unescape chars mentioned in RFC4515. """ + if encoding is None: + encoding = get_config_parameter('DEFAULT_ENCODING') + + unescaped = to_raw(text, encoding) + unescaped = unescaped.replace(b'\\5c', b'\\') + unescaped = unescaped.replace(b'\\5C', b'\\') + unescaped = unescaped.replace(b'\\2a', b'*') + unescaped = unescaped.replace(b'\\2A', b'*') + unescaped = unescaped.replace(b'\\28', b'(') + unescaped = unescaped.replace(b'\\29', b')') + unescaped = unescaped.replace(b'\\00', b'\x00') + return unescaped + + +def escape_bytes(bytes_value): + """ Convert a byte sequence to a properly escaped for LDAP (format BACKSLASH HEX HEX) string""" + if bytes_value: + if str is not bytes: # Python 3 + if isinstance(bytes_value, str): + bytes_value = bytearray(bytes_value, encoding='utf-8') + escaped = '\\'.join([('%02x' % int(b)) for b in bytes_value]) + else: # Python 2 + if isinstance(bytes_value, unicode): + bytes_value = bytes_value.encode('utf-8') + escaped = '\\'.join([('%02x' % ord(b)) for b in bytes_value]) + else: + escaped = '' + + return ('\\' + escaped) if escaped else '' + + +def prepare_for_stream(value): + if str is not bytes: # Python 3 + return value + else: # Python 2 + return value.decode() + + +def json_encode_b64(obj): + try: + return dict(encoding='base64', encoded=b64encode(obj)) + except Exception as e: + raise LDAPDefinitionError('unable to encode ' + str(obj) + ' - ' + str(e)) + + +# noinspection PyProtectedMember +def check_json_dict(json_dict): + # needed for python 2 + + for k, v in json_dict.items(): + if isinstance(v, dict): + check_json_dict(v) + elif isinstance(v, CaseInsensitiveDict): + check_json_dict(v._store) + elif isinstance(v, SEQUENCE_TYPES): + for i, e in enumerate(v): + if isinstance(e, dict): + check_json_dict(e) + elif isinstance(e, CaseInsensitiveDict): + check_json_dict(e._store) + else: + v[i] = format_json(e) + else: + json_dict[k] = format_json(v) + + +def json_hook(obj): + if hasattr(obj, 'keys') and len(list(obj.keys())) == 2 and 'encoding' in obj.keys() and 'encoded' in obj.keys(): + return b64decode(obj['encoded']) + + return obj + + +# noinspection PyProtectedMember +def format_json(obj): + if isinstance(obj, CaseInsensitiveDict): + return obj._store + + if isinstance(obj, datetime.datetime): + return str(obj) + + if isinstance(obj, int): + return obj + + if isinstance(obj, datetime.timedelta): + return str(obj) + + if str is bytes: # Python 2 + if isinstance(obj, long): # long exists only in python2 + return obj + + try: + if str is not bytes: # Python 3 + if isinstance(obj, bytes): + # return check_escape(str(obj, 'utf-8', errors='strict')) + return str(obj, 'utf-8', errors='strict') + raise LDAPDefinitionError('unable to serialize ' + str(obj)) + else: # Python 2 + if isinstance(obj, unicode): + return obj + else: + # return unicode(check_escape(obj)) + return unicode(obj) + except (TypeError, UnicodeDecodeError): + pass + + try: + return json_encode_b64(bytes(obj)) + except Exception: + pass + + raise LDAPDefinitionError('unable to serialize ' + str(obj)) + + +def is_filter_escaped(text): + if not type(text) == ((str is not bytes) and str or unicode): # requires str for Python 3 and unicode for Python 2 + raise ValueError('unicode input expected') + + return all(c not in text for c in '()*\0') and not re.search('\\\\([^0-9a-fA-F]|(.[^0-9a-fA-F]))', text) + + +def ldap_escape_to_bytes(text): + bytesequence = bytearray() + i = 0 + try: + if isinstance(text, STRING_TYPES): + while i < len(text): + if text[i] == '\\': + if len(text) > i + 2: + try: + bytesequence.append(int(text[i+1:i+3], 16)) + i += 3 + continue + except ValueError: + pass + bytesequence.append(92) # "\" ASCII code + else: + raw = to_raw(text[i]) + for c in raw: + bytesequence.append(c) + i += 1 + elif isinstance(text, (bytes, bytearray)): + while i < len(text): + if text[i] == 92: # "\" ASCII code + if len(text) > i + 2: + try: + bytesequence.append(int(text[i + 1:i + 3], 16)) + i += 3 + continue + except ValueError: + pass + bytesequence.append(92) # "\" ASCII code + else: + bytesequence.append(text[i]) + i += 1 + except Exception: + raise LDAPDefinitionError('badly formatted LDAP byte escaped sequence') + + return bytes(bytesequence) diff --git a/server/www/packages/packages-linux/x64/ldap3/utils/dn.py b/server/www/packages/packages-linux/x64/ldap3/utils/dn.py index d1a50a9..99705ce 100644 --- a/server/www/packages/packages-linux/x64/ldap3/utils/dn.py +++ b/server/www/packages/packages-linux/x64/ldap3/utils/dn.py @@ -1,375 +1,403 @@ -""" -""" - -# Created on 2014.09.08 -# -# Author: Giovanni Cannata -# -# Copyright 2014 - 2018 Giovanni Cannata -# -# This file is part of ldap3. -# -# ldap3 is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# ldap3 is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with ldap3 in the COPYING and COPYING.LESSER files. -# If not, see . - -from string import hexdigits, ascii_letters, digits - -from .. import SEQUENCE_TYPES -from ..core.exceptions import LDAPInvalidDnError - - -STATE_ANY = 0 -STATE_ESCAPE = 1 -STATE_ESCAPE_HEX = 2 - - -def _add_ava(ava, decompose, remove_space, space_around_equal): - if not ava: - return '' - - space = ' ' if space_around_equal else '' - attr_name, _, value = ava.partition('=') - if decompose: - if remove_space: - component = (attr_name.strip(), value.strip()) - else: - component = (attr_name, value) - else: - if remove_space: - component = attr_name.strip() + space + '=' + space + value.strip() - else: - component = attr_name + space + '=' + space + value - - return component - - -def to_dn(iterator, decompose=False, remove_space=False, space_around_equal=False, separate_rdn=False): - """ - Convert an iterator to a list of dn parts - if decompose=True return a list of tuple (one for each dn component) else return a list of strings - if remove_space=True removes unneeded spaces - if space_around_equal=True add spaces around equal in returned strings - if separate_rdn=True consider multiple RDNs as different component of DN - """ - dn = [] - component = '' - escape_sequence = False - for c in iterator: - if c == '\\': # escape sequence - escape_sequence = True - elif escape_sequence and c != ' ': - escape_sequence = False - elif c == '+' and separate_rdn: - dn.append(_add_ava(component, decompose, remove_space, space_around_equal)) - component = '' - continue - elif c == ',': - if '=' in component: - dn.append(_add_ava(component, decompose, remove_space, space_around_equal)) - component = '' - continue - - component += c - - dn.append(_add_ava(component, decompose, remove_space, space_around_equal)) - return dn - - -def _find_first_unescaped(dn, char, pos): - while True: - pos = dn.find(char, pos) - if pos == -1: - break # no char found - if pos > 0 and dn[pos - 1] != '\\': # unescaped char - break - - pos += 1 - - return pos - - -def _find_last_unescaped(dn, char, start, stop=0): - while True: - stop = dn.rfind(char, start, stop) - if stop == -1: - break - if stop >= 0 and dn[stop - 1] != '\\': - break - - if stop < start: - stop = -1 - break - - return stop - - -def _get_next_ava(dn): - comma = _find_first_unescaped(dn, ',', 0) - plus = _find_first_unescaped(dn, '+', 0) - - if plus > 0 and (plus < comma or comma == -1): - equal = _find_first_unescaped(dn, '=', plus + 1) - if equal > plus + 1: - plus = _find_last_unescaped(dn, '+', plus, equal) - return dn[:plus], '+' - - if comma > 0: - equal = _find_first_unescaped(dn, '=', comma + 1) - if equal > comma + 1: - comma = _find_last_unescaped(dn, ',', comma, equal) - return dn[:comma], ',' - - return dn, '' - - -def _split_ava(ava, escape=False, strip=True): - equal = ava.find('=') - while equal > 0: # not first character - if ava[equal - 1] != '\\': # not an escaped equal so it must be an ava separator - # attribute_type1 = ava[0:equal].strip() if strip else ava[0:equal] - if strip: - attribute_type = ava[0:equal].strip() - attribute_value = _escape_attribute_value(ava[equal + 1:].strip()) if escape else ava[equal + 1:].strip() - else: - attribute_type = ava[0:equal] - attribute_value = _escape_attribute_value(ava[equal + 1:]) if escape else ava[equal + 1:] - - return attribute_type, attribute_value - equal = ava.find('=', equal + 1) - - return '', (ava.strip if strip else ava) # if no equal found return only value - - -def _validate_attribute_type(attribute_type): - if not attribute_type: - raise LDAPInvalidDnError('attribute type not present') - - if attribute_type == ' pairs') - if attribute_value[0] == ' ': # space cannot be used as first or last character - raise LDAPInvalidDnError('SPACE not allowed as first character of attribute value') - if attribute_value[-1] == ' ': - raise LDAPInvalidDnError('SPACE not allowed as last character of attribute value') - - state = STATE_ANY - for c in attribute_value: - if state == STATE_ANY: - if c == '\\': - state = STATE_ESCAPE - elif c in '"#+,;<=>\00': - raise LDAPInvalidDnError('special characters ' + c + ' must be escaped') - elif state == STATE_ESCAPE: - if c in hexdigits: - state = STATE_ESCAPE_HEX - elif c in ' "#+,;<=>\\\00': - state = STATE_ANY - else: - raise LDAPInvalidDnError('invalid escaped character ' + c) - elif state == STATE_ESCAPE_HEX: - if c in hexdigits: - state = STATE_ANY - else: - raise LDAPInvalidDnError('invalid escaped character ' + c) - - # final state - if state != STATE_ANY: - raise LDAPInvalidDnError('invalid final character') - - return True - - -def _escape_attribute_value(attribute_value): - if not attribute_value: - return '' - - if attribute_value[0] == '#': # with leading SHARP only pairs of hex characters are valid - valid_hex = True - if len(attribute_value) % 2 == 0: # string must be # + HEX HEX (an odd number of chars) - valid_hex = False - - if valid_hex: - for c in attribute_value: - if c not in hexdigits: # allowed only hex digits as per RFC 4514 - valid_hex = False - break - - if valid_hex: - return attribute_value - - state = STATE_ANY - escaped = '' - tmp_buffer = '' - for c in attribute_value: - if state == STATE_ANY: - if c == '\\': - state = STATE_ESCAPE - elif c in '"#+,;<=>\00': - escaped += '\\' + c - else: - escaped += c - elif state == STATE_ESCAPE: - if c in hexdigits: - tmp_buffer = c - state = STATE_ESCAPE_HEX - elif c in ' "#+,;<=>\\\00': - escaped += '\\' + c - state = STATE_ANY - else: - escaped += '\\\\' + c - elif state == STATE_ESCAPE_HEX: - if c in hexdigits: - escaped += '\\' + tmp_buffer + c - else: - escaped += '\\\\' + tmp_buffer + c - tmp_buffer = '' - state = STATE_ANY - - # final state - if state == STATE_ESCAPE: - escaped += '\\\\' - elif state == STATE_ESCAPE_HEX: - escaped += '\\\\' + tmp_buffer - - if escaped[0] == ' ': # leading SPACE must be escaped - escaped = '\\' + escaped - - if escaped[-1] == ' ' and len(escaped) > 1 and escaped[-2] != '\\': # trailing SPACE must be escaped - escaped = escaped[:-1] + '\\ ' - - return escaped - - -def parse_dn(dn, escape=False, strip=True): - rdns = [] - avas = [] - while dn: - ava, separator = _get_next_ava(dn) # if returned ava doesn't containg any unescaped equal it'a appended to last ava in avas - - dn = dn[len(ava) + 1:] - if _find_first_unescaped(ava, '=', 0) > 0 or len(avas) == 0: - avas.append((ava, separator)) - else: - avas[len(avas) - 1] = (avas[len(avas) - 1][0] + avas[len(avas) - 1][1] + ava, separator) - - for ava, separator in avas: - attribute_type, attribute_value = _split_ava(ava, escape, strip) - - if not _validate_attribute_type(attribute_type): - raise LDAPInvalidDnError('unable to validate attribute type in ' + ava) - - if not _validate_attribute_value(attribute_value): - raise LDAPInvalidDnError('unable to validate attribute value in ' + ava) - - rdns.append((attribute_type, attribute_value, separator)) - dn = dn[len(ava) + 1:] - - if not rdns: - raise LDAPInvalidDnError('empty dn') - - return rdns - - -def safe_dn(dn, decompose=False, reverse=False): - """ - normalize and escape a dn, if dn is a sequence it is joined. - the reverse parameter changes the join direction of the sequence - """ - if isinstance(dn, SEQUENCE_TYPES): - components = [rdn for rdn in dn] - if reverse: - dn = ','.join(reversed(components)) - else: - dn = ','.join(components) - if decompose: - escaped_dn = [] - else: - escaped_dn = '' - - if dn.startswith(''): # Active Directory allows looking up objects by putting its GUID in a specially-formatted DN (e.g. '') - escaped_dn = dn - elif '@' not in dn and '\\' not in dn: # active directory UPN (User Principal Name) consist of an account, the at sign (@) and a domain, or the domain level logn name domain\username - for component in parse_dn(dn, escape=True): - if decompose: - escaped_dn.append((component[0], component[1], component[2])) - else: - escaped_dn += component[0] + '=' + component[1] + component[2] - elif '@' in dn and '=' not in dn and len(dn.split('@')) != 2: - raise LDAPInvalidDnError('Active Directory User Principal Name must consist of name@domain') - elif '\\' in dn and '=' not in dn and len(dn.split('\\')) != 2: - raise LDAPInvalidDnError('Active Directory Domain Level Logon Name must consist of name\\domain') - else: - escaped_dn = dn - - return escaped_dn - - -def safe_rdn(dn, decompose=False): - """Returns a list of rdn for the dn, usually there is only one rdn, but it can be more than one when the + sign is used""" - escaped_rdn = [] - one_more = True - for component in parse_dn(dn, escape=True): - if component[2] == '+' or one_more: - if decompose: - escaped_rdn.append((component[0], component[1])) - else: - escaped_rdn.append(component[0] + '=' + component[1]) - if component[2] == '+': - one_more = True - else: - one_more = False - break - - if one_more: - raise LDAPInvalidDnError('bad dn ' + str(dn)) - - return escaped_rdn - - -def escape_rdn(rdn): - """ - Escape rdn characters to prevent injection according to RFC 4514. - """ - - # '/' must be handled first or the escape slashes will be escaped! - for char in ['\\', ',', '+', '"', '<', '>', ';', '=', '\x00']: - rdn = rdn.replace(char, '\\' + char) - - if rdn[0] == '#' or rdn[0] == ' ': - rdn = ''.join(('\\', rdn)) - - if rdn[-1] == ' ': - rdn = ''.join((rdn[:-1], '\\ ')) - - return rdn +""" +""" + +# Created on 2014.09.08 +# +# Author: Giovanni Cannata +# +# Copyright 2014 - 2019 Giovanni Cannata +# +# This file is part of ldap3. +# +# ldap3 is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ldap3 is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with ldap3 in the COPYING and COPYING.LESSER files. +# If not, see . + +from string import hexdigits, ascii_letters, digits + +from .. import SEQUENCE_TYPES +from ..core.exceptions import LDAPInvalidDnError + + +STATE_ANY = 0 +STATE_ESCAPE = 1 +STATE_ESCAPE_HEX = 2 + + +def _add_ava(ava, decompose, remove_space, space_around_equal): + if not ava: + return '' + + space = ' ' if space_around_equal else '' + attr_name, _, value = ava.partition('=') + if decompose: + if remove_space: + component = (attr_name.strip(), value.strip()) + else: + component = (attr_name, value) + else: + if remove_space: + component = attr_name.strip() + space + '=' + space + value.strip() + else: + component = attr_name + space + '=' + space + value + + return component + + +def to_dn(iterator, decompose=False, remove_space=False, space_around_equal=False, separate_rdn=False): + """ + Convert an iterator to a list of dn parts + if decompose=True return a list of tuple (one for each dn component) else return a list of strings + if remove_space=True removes unneeded spaces + if space_around_equal=True add spaces around equal in returned strings + if separate_rdn=True consider multiple RDNs as different component of DN + """ + dn = [] + component = '' + escape_sequence = False + for c in iterator: + if c == '\\': # escape sequence + escape_sequence = True + elif escape_sequence and c != ' ': + escape_sequence = False + elif c == '+' and separate_rdn: + dn.append(_add_ava(component, decompose, remove_space, space_around_equal)) + component = '' + continue + elif c == ',': + if '=' in component: + dn.append(_add_ava(component, decompose, remove_space, space_around_equal)) + component = '' + continue + + component += c + + dn.append(_add_ava(component, decompose, remove_space, space_around_equal)) + return dn + + +def _find_first_unescaped(dn, char, pos): + while True: + pos = dn.find(char, pos) + if pos == -1: + break # no char found + if pos > 0 and dn[pos - 1] != '\\': # unescaped char + break + elif pos > 1 and dn[pos - 1] == '\\': # may be unescaped + escaped = True + for c in dn[pos - 2:0:-1]: + if c == '\\': + escaped = not escaped + else: + break + if not escaped: + break + pos += 1 + + return pos + + +def _find_last_unescaped(dn, char, start, stop=0): + while True: + stop = dn.rfind(char, start, stop) + if stop == -1: + break + if stop >= 0 and dn[stop - 1] != '\\': + break + elif stop > 1 and dn[stop - 1] == '\\': # may be unescaped + escaped = True + for c in dn[stop - 2:0:-1]: + if c == '\\': + escaped = not escaped + else: + break + if not escaped: + break + if stop < start: + stop = -1 + break + + return stop + + +def _get_next_ava(dn): + comma = _find_first_unescaped(dn, ',', 0) + plus = _find_first_unescaped(dn, '+', 0) + + if plus > 0 and (plus < comma or comma == -1): + equal = _find_first_unescaped(dn, '=', plus + 1) + if equal > plus + 1: + plus = _find_last_unescaped(dn, '+', plus, equal) + return dn[:plus], '+' + + if comma > 0: + equal = _find_first_unescaped(dn, '=', comma + 1) + if equal > comma + 1: + comma = _find_last_unescaped(dn, ',', comma, equal) + return dn[:comma], ',' + + return dn, '' + + +def _split_ava(ava, escape=False, strip=True): + equal = ava.find('=') + while equal > 0: # not first character + if ava[equal - 1] != '\\': # not an escaped equal so it must be an ava separator + # attribute_type1 = ava[0:equal].strip() if strip else ava[0:equal] + if strip: + attribute_type = ava[0:equal].strip() + attribute_value = _escape_attribute_value(ava[equal + 1:].strip()) if escape else ava[equal + 1:].strip() + else: + attribute_type = ava[0:equal] + attribute_value = _escape_attribute_value(ava[equal + 1:]) if escape else ava[equal + 1:] + + return attribute_type, attribute_value + equal = ava.find('=', equal + 1) + + return '', (ava.strip if strip else ava) # if no equal found return only value + + +def _validate_attribute_type(attribute_type): + if not attribute_type: + raise LDAPInvalidDnError('attribute type not present') + + if attribute_type == ' pairs') + if attribute_value[0] == ' ': # unescaped space cannot be used as leading or last character + raise LDAPInvalidDnError('SPACE must be escaped as leading character of attribute value') + if attribute_value.endswith(' ') and not attribute_value.endswith('\\ '): + raise LDAPInvalidDnError('SPACE must be escaped as trailing character of attribute value') + + state = STATE_ANY + for c in attribute_value: + if state == STATE_ANY: + if c == '\\': + state = STATE_ESCAPE + elif c in '"#+,;<=>\00': + raise LDAPInvalidDnError('special character ' + c + ' must be escaped') + elif state == STATE_ESCAPE: + if c in hexdigits: + state = STATE_ESCAPE_HEX + elif c in ' "#+,;<=>\\\00': + state = STATE_ANY + else: + raise LDAPInvalidDnError('invalid escaped character ' + c) + elif state == STATE_ESCAPE_HEX: + if c in hexdigits: + state = STATE_ANY + else: + raise LDAPInvalidDnError('invalid escaped character ' + c) + + # final state + if state != STATE_ANY: + raise LDAPInvalidDnError('invalid final character') + + return True + + +def _escape_attribute_value(attribute_value): + if not attribute_value: + return '' + + if attribute_value[0] == '#': # with leading SHARP only pairs of hex characters are valid + valid_hex = True + if len(attribute_value) % 2 == 0: # string must be # + HEX HEX (an odd number of chars) + valid_hex = False + + if valid_hex: + for c in attribute_value: + if c not in hexdigits: # allowed only hex digits as per RFC 4514 + valid_hex = False + break + + if valid_hex: + return attribute_value + + state = STATE_ANY + escaped = '' + tmp_buffer = '' + for c in attribute_value: + if state == STATE_ANY: + if c == '\\': + state = STATE_ESCAPE + elif c in '"#+,;<=>\00': + escaped += '\\' + c + else: + escaped += c + elif state == STATE_ESCAPE: + if c in hexdigits: + tmp_buffer = c + state = STATE_ESCAPE_HEX + elif c in ' "#+,;<=>\\\00': + escaped += '\\' + c + state = STATE_ANY + else: + escaped += '\\\\' + c + elif state == STATE_ESCAPE_HEX: + if c in hexdigits: + escaped += '\\' + tmp_buffer + c + else: + escaped += '\\\\' + tmp_buffer + c + tmp_buffer = '' + state = STATE_ANY + + # final state + if state == STATE_ESCAPE: + escaped += '\\\\' + elif state == STATE_ESCAPE_HEX: + escaped += '\\\\' + tmp_buffer + + if escaped[0] == ' ': # leading SPACE must be escaped + escaped = '\\' + escaped + + if escaped[-1] == ' ' and len(escaped) > 1 and escaped[-2] != '\\': # trailing SPACE must be escaped + escaped = escaped[:-1] + '\\ ' + + return escaped + + +def parse_dn(dn, escape=False, strip=False): + """ + Parses a DN into syntactic components + :param dn: + :param escape: + :param strip: + :return: + a list of tripels representing `attributeTypeAndValue` elements + containing `attributeType`, `attributeValue` and the following separator (`COMMA` or `PLUS`) if given, else an empty `str`. + in their original representation, still containing escapes or encoded as hex. + """ + rdns = [] + avas = [] + while dn: + ava, separator = _get_next_ava(dn) # if returned ava doesn't containg any unescaped equal it'a appended to last ava in avas + + dn = dn[len(ava) + 1:] + if _find_first_unescaped(ava, '=', 0) > 0 or len(avas) == 0: + avas.append((ava, separator)) + else: + avas[len(avas) - 1] = (avas[len(avas) - 1][0] + avas[len(avas) - 1][1] + ava, separator) + + for ava, separator in avas: + attribute_type, attribute_value = _split_ava(ava, escape, strip) + + if not _validate_attribute_type(attribute_type): + raise LDAPInvalidDnError('unable to validate attribute type in ' + ava) + + if not _validate_attribute_value(attribute_value): + raise LDAPInvalidDnError('unable to validate attribute value in ' + ava) + + rdns.append((attribute_type, attribute_value, separator)) + dn = dn[len(ava) + 1:] + + if not rdns: + raise LDAPInvalidDnError('empty dn') + + return rdns + + +def safe_dn(dn, decompose=False, reverse=False): + """ + normalize and escape a dn, if dn is a sequence it is joined. + the reverse parameter changes the join direction of the sequence + """ + if isinstance(dn, SEQUENCE_TYPES): + components = [rdn for rdn in dn] + if reverse: + dn = ','.join(reversed(components)) + else: + dn = ','.join(components) + if decompose: + escaped_dn = [] + else: + escaped_dn = '' + + if dn.startswith(''): # Active Directory allows looking up objects by putting its GUID in a specially-formatted DN (e.g. '') + escaped_dn = dn + elif dn.startswith(''): # Active Directory allows Binding to Well-Known Objects Using WKGUID in a specially-formatted DN (e.g. ) + escaped_dn = dn + elif '@' not in dn: # active directory UPN (User Principal Name) consist of an account, the at sign (@) and a domain, or the domain level logn name domain\username + for component in parse_dn(dn, escape=True): + if decompose: + escaped_dn.append((component[0], component[1], component[2])) + else: + escaped_dn += component[0] + '=' + component[1] + component[2] + elif '@' in dn and '=' not in dn and len(dn.split('@')) != 2: + raise LDAPInvalidDnError('Active Directory User Principal Name must consist of name@domain') + elif '\\' in dn and '=' not in dn and len(dn.split('\\')) != 2: + raise LDAPInvalidDnError('Active Directory Domain Level Logon Name must consist of name\\domain') + else: + escaped_dn = dn + + return escaped_dn + + +def safe_rdn(dn, decompose=False): + """Returns a list of rdn for the dn, usually there is only one rdn, but it can be more than one when the + sign is used""" + escaped_rdn = [] + one_more = True + for component in parse_dn(dn, escape=True): + if component[2] == '+' or one_more: + if decompose: + escaped_rdn.append((component[0], component[1])) + else: + escaped_rdn.append(component[0] + '=' + component[1]) + if component[2] == '+': + one_more = True + else: + one_more = False + break + + if one_more: + raise LDAPInvalidDnError('bad dn ' + str(dn)) + + return escaped_rdn + + +def escape_rdn(rdn): + """ + Escape rdn characters to prevent injection according to RFC 4514. + """ + + # '/' must be handled first or the escape slashes will be escaped! + for char in ['\\', ',', '+', '"', '<', '>', ';', '=', '\x00']: + rdn = rdn.replace(char, '\\' + char) + + if rdn[0] == '#' or rdn[0] == ' ': + rdn = ''.join(('\\', rdn)) + + if rdn[-1] == ' ': + rdn = ''.join((rdn[:-1], '\\ ')) + + return rdn diff --git a/server/www/packages/packages-linux/x64/ldap3/utils/ntlm.py b/server/www/packages/packages-linux/x64/ldap3/utils/ntlm.py index 54efaae..af32a87 100644 --- a/server/www/packages/packages-linux/x64/ldap3/utils/ntlm.py +++ b/server/www/packages/packages-linux/x64/ldap3/utils/ntlm.py @@ -483,7 +483,7 @@ class NtlmClient(object): temp += self.server_target_info_raw temp += pack(' +# Copyright 2006-2019 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -__version__ = '1.0.7' +__version__ = "1.1.0" diff --git a/server/www/packages/packages-linux/x64/mako/_ast_util.py b/server/www/packages/packages-linux/x64/mako/_ast_util.py index c410287..74c0851 100644 --- a/server/www/packages/packages-linux/x64/mako/_ast_util.py +++ b/server/www/packages/packages-linux/x64/mako/_ast_util.py @@ -1,5 +1,5 @@ # mako/_ast_util.py -# Copyright (C) 2006-2016 the Mako authors and contributors +# Copyright 2006-2019 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -8,69 +8,77 @@ ast ~~~ - The `ast` module helps Python applications to process trees of the Python - abstract syntax grammar. The abstract syntax itself might change with - each Python release; this module helps to find out programmatically what - the current grammar looks like and allows modifications of it. - - An abstract syntax tree can be generated by passing `ast.PyCF_ONLY_AST` as - a flag to the `compile()` builtin function or by using the `parse()` - function from this module. The result will be a tree of objects whose - classes all inherit from `ast.AST`. - - A modified abstract syntax tree can be compiled into a Python code object - using the built-in `compile()` function. - - Additionally various helper functions are provided that make working with - the trees simpler. The main intention of the helper functions and this - module in general is to provide an easy to use interface for libraries - that work tightly with the python syntax (template engines for example). - + This is a stripped down version of Armin Ronacher's ast module. :copyright: Copyright 2008 by Armin Ronacher. :license: Python License. """ -from _ast import * # noqa + + +from _ast import Add +from _ast import And +from _ast import AST +from _ast import BitAnd +from _ast import BitOr +from _ast import BitXor +from _ast import Div +from _ast import Eq +from _ast import FloorDiv +from _ast import Gt +from _ast import GtE +from _ast import If +from _ast import In +from _ast import Invert +from _ast import Is +from _ast import IsNot +from _ast import LShift +from _ast import Lt +from _ast import LtE +from _ast import Mod +from _ast import Mult +from _ast import Name +from _ast import Not +from _ast import NotEq +from _ast import NotIn +from _ast import Or +from _ast import PyCF_ONLY_AST +from _ast import RShift +from _ast import Sub +from _ast import UAdd +from _ast import USub + from mako.compat import arg_stringname -BOOLOP_SYMBOLS = { - And: 'and', - Or: 'or' -} +BOOLOP_SYMBOLS = {And: "and", Or: "or"} BINOP_SYMBOLS = { - Add: '+', - Sub: '-', - Mult: '*', - Div: '/', - FloorDiv: '//', - Mod: '%', - LShift: '<<', - RShift: '>>', - BitOr: '|', - BitAnd: '&', - BitXor: '^' + Add: "+", + Sub: "-", + Mult: "*", + Div: "/", + FloorDiv: "//", + Mod: "%", + LShift: "<<", + RShift: ">>", + BitOr: "|", + BitAnd: "&", + BitXor: "^", } CMPOP_SYMBOLS = { - Eq: '==', - Gt: '>', - GtE: '>=', - In: 'in', - Is: 'is', - IsNot: 'is not', - Lt: '<', - LtE: '<=', - NotEq: '!=', - NotIn: 'not in' + Eq: "==", + Gt: ">", + GtE: ">=", + In: "in", + Is: "is", + IsNot: "is not", + Lt: "<", + LtE: "<=", + NotEq: "!=", + NotIn: "not in", } -UNARYOP_SYMBOLS = { - Invert: '~', - Not: 'not', - UAdd: '+', - USub: '-' -} +UNARYOP_SYMBOLS = {Invert: "~", Not: "not", UAdd: "+", USub: "-"} ALL_SYMBOLS = {} ALL_SYMBOLS.update(BOOLOP_SYMBOLS) @@ -79,105 +87,15 @@ ALL_SYMBOLS.update(CMPOP_SYMBOLS) ALL_SYMBOLS.update(UNARYOP_SYMBOLS) -def parse(expr, filename='', mode='exec'): +def parse(expr, filename="", mode="exec"): """Parse an expression into an AST node.""" return compile(expr, filename, mode, PyCF_ONLY_AST) -def to_source(node, indent_with=' ' * 4): - """ - This function can convert a node tree back into python sourcecode. This - is useful for debugging purposes, especially if you're dealing with custom - asts not generated by python itself. - - It could be that the sourcecode is evaluable when the AST itself is not - compilable / evaluable. The reason for this is that the AST contains some - more data than regular sourcecode does, which is dropped during - conversion. - - Each level of indentation is replaced with `indent_with`. Per default this - parameter is equal to four spaces as suggested by PEP 8, but it might be - adjusted to match the application's styleguide. - """ - generator = SourceGenerator(indent_with) - generator.visit(node) - return ''.join(generator.result) - - -def dump(node): - """ - A very verbose representation of the node passed. This is useful for - debugging purposes. - """ - def _format(node): - if isinstance(node, AST): - return '%s(%s)' % (node.__class__.__name__, - ', '.join('%s=%s' % (a, _format(b)) - for a, b in iter_fields(node))) - elif isinstance(node, list): - return '[%s]' % ', '.join(_format(x) for x in node) - return repr(node) - if not isinstance(node, AST): - raise TypeError('expected AST, got %r' % node.__class__.__name__) - return _format(node) - - -def copy_location(new_node, old_node): - """ - Copy the source location hint (`lineno` and `col_offset`) from the - old to the new node if possible and return the new one. - """ - for attr in 'lineno', 'col_offset': - if attr in old_node._attributes and attr in new_node._attributes \ - and hasattr(old_node, attr): - setattr(new_node, attr, getattr(old_node, attr)) - return new_node - - -def fix_missing_locations(node): - """ - Some nodes require a line number and the column offset. Without that - information the compiler will abort the compilation. Because it can be - a dull task to add appropriate line numbers and column offsets when - adding new nodes this function can help. It copies the line number and - column offset of the parent node to the child nodes without this - information. - - Unlike `copy_location` this works recursive and won't touch nodes that - already have a location information. - """ - def _fix(node, lineno, col_offset): - if 'lineno' in node._attributes: - if not hasattr(node, 'lineno'): - node.lineno = lineno - else: - lineno = node.lineno - if 'col_offset' in node._attributes: - if not hasattr(node, 'col_offset'): - node.col_offset = col_offset - else: - col_offset = node.col_offset - for child in iter_child_nodes(node): - _fix(child, lineno, col_offset) - _fix(node, 1, 0) - return node - - -def increment_lineno(node, n=1): - """ - Increment the line numbers of all nodes by `n` if they have line number - attributes. This is useful to "move code" to a different location in a - file. - """ - for node in zip((node,), walk(node)): - if 'lineno' in node._attributes: - node.lineno = getattr(node, 'lineno', 0) + n - - def iter_fields(node): """Iterate over all fields of a node, only yielding existing fields.""" # CPython 2.5 compat - if not hasattr(node, '_fields') or not node._fields: + if not hasattr(node, "_fields") or not node._fields: return for field in node._fields: try: @@ -186,65 +104,6 @@ def iter_fields(node): pass -def get_fields(node): - """Like `iter_fields` but returns a dict.""" - return dict(iter_fields(node)) - - -def iter_child_nodes(node): - """Iterate over all child nodes or a node.""" - for name, field in iter_fields(node): - if isinstance(field, AST): - yield field - elif isinstance(field, list): - for item in field: - if isinstance(item, AST): - yield item - - -def get_child_nodes(node): - """Like `iter_child_nodes` but returns a list.""" - return list(iter_child_nodes(node)) - - -def get_compile_mode(node): - """ - Get the mode for `compile` of a given node. If the node is not a `mod` - node (`Expression`, `Module` etc.) a `TypeError` is thrown. - """ - if not isinstance(node, mod): - raise TypeError('expected mod node, got %r' % node.__class__.__name__) - return { - Expression: 'eval', - Interactive: 'single' - }.get(node.__class__, 'expr') - - -def get_docstring(node): - """ - Return the docstring for the given node or `None` if no docstring can be - found. If the node provided does not accept docstrings a `TypeError` - will be raised. - """ - if not isinstance(node, (FunctionDef, ClassDef, Module)): - raise TypeError("%r can't have docstrings" % node.__class__.__name__) - if node.body and isinstance(node.body[0], Str): - return node.body[0].s - - -def walk(node): - """ - Iterate over all nodes. This is useful if you only want to modify nodes in - place and don't care about the context or the order the nodes are returned. - """ - from collections import deque - todo = deque([node]) - while todo: - node = todo.popleft() - todo.extend(iter_child_nodes(node)) - yield node - - class NodeVisitor(object): """ @@ -269,7 +128,7 @@ class NodeVisitor(object): exists for this node. In that case the generic visit function is used instead. """ - method = 'visit_' + node.__class__.__name__ + method = "visit_" + node.__class__.__name__ return getattr(self, method, None) def visit(self, node): @@ -367,7 +226,7 @@ class SourceGenerator(NodeVisitor): def write(self, x): if self.new_lines: if self.result: - self.result.append('\n' * self.new_lines) + self.result.append("\n" * self.new_lines) self.result.append(self.indent_with * self.indentation) self.new_lines = 0 self.result.append(x) @@ -386,7 +245,7 @@ class SourceGenerator(NodeVisitor): self.body(node.body) if node.orelse: self.newline() - self.write('else:') + self.write("else:") self.body(node.orelse) def signature(self, node): @@ -394,7 +253,7 @@ class SourceGenerator(NodeVisitor): def write_comma(): if want_comma: - self.write(', ') + self.write(", ") else: want_comma.append(True) @@ -403,19 +262,19 @@ class SourceGenerator(NodeVisitor): write_comma() self.visit(arg) if default is not None: - self.write('=') + self.write("=") self.visit(default) if node.vararg is not None: write_comma() - self.write('*' + arg_stringname(node.vararg)) + self.write("*" + arg_stringname(node.vararg)) if node.kwarg is not None: write_comma() - self.write('**' + arg_stringname(node.kwarg)) + self.write("**" + arg_stringname(node.kwarg)) def decorators(self, node): for decorator in node.decorator_list: self.newline() - self.write('@') + self.write("@") self.visit(decorator) # Statements @@ -424,29 +283,29 @@ class SourceGenerator(NodeVisitor): self.newline() for idx, target in enumerate(node.targets): if idx: - self.write(', ') + self.write(", ") self.visit(target) - self.write(' = ') + self.write(" = ") self.visit(node.value) def visit_AugAssign(self, node): self.newline() self.visit(node.target) - self.write(BINOP_SYMBOLS[type(node.op)] + '=') + self.write(BINOP_SYMBOLS[type(node.op)] + "=") self.visit(node.value) def visit_ImportFrom(self, node): self.newline() - self.write('from %s%s import ' % ('.' * node.level, node.module)) + self.write("from %s%s import " % ("." * node.level, node.module)) for idx, item in enumerate(node.names): if idx: - self.write(', ') + self.write(", ") self.write(item) def visit_Import(self, node): self.newline() for item in node.names: - self.write('import ') + self.write("import ") self.visit(item) def visit_Expr(self, node): @@ -457,9 +316,9 @@ class SourceGenerator(NodeVisitor): self.newline(n=2) self.decorators(node) self.newline() - self.write('def %s(' % node.name) + self.write("def %s(" % node.name) self.signature(node.args) - self.write('):') + self.write("):") self.body(node.body) def visit_ClassDef(self, node): @@ -467,200 +326,200 @@ class SourceGenerator(NodeVisitor): def paren_or_comma(): if have_args: - self.write(', ') + self.write(", ") else: have_args.append(True) - self.write('(') + self.write("(") self.newline(n=3) self.decorators(node) self.newline() - self.write('class %s' % node.name) + self.write("class %s" % node.name) for base in node.bases: paren_or_comma() self.visit(base) # XXX: the if here is used to keep this module compatible # with python 2.6. - if hasattr(node, 'keywords'): + if hasattr(node, "keywords"): for keyword in node.keywords: paren_or_comma() - self.write(keyword.arg + '=') + self.write(keyword.arg + "=") self.visit(keyword.value) if getattr(node, "starargs", None): paren_or_comma() - self.write('*') + self.write("*") self.visit(node.starargs) if getattr(node, "kwargs", None): paren_or_comma() - self.write('**') + self.write("**") self.visit(node.kwargs) - self.write(have_args and '):' or ':') + self.write(have_args and "):" or ":") self.body(node.body) def visit_If(self, node): self.newline() - self.write('if ') + self.write("if ") self.visit(node.test) - self.write(':') + self.write(":") self.body(node.body) while True: else_ = node.orelse if len(else_) == 1 and isinstance(else_[0], If): node = else_[0] self.newline() - self.write('elif ') + self.write("elif ") self.visit(node.test) - self.write(':') + self.write(":") self.body(node.body) else: self.newline() - self.write('else:') + self.write("else:") self.body(else_) break def visit_For(self, node): self.newline() - self.write('for ') + self.write("for ") self.visit(node.target) - self.write(' in ') + self.write(" in ") self.visit(node.iter) - self.write(':') + self.write(":") self.body_or_else(node) def visit_While(self, node): self.newline() - self.write('while ') + self.write("while ") self.visit(node.test) - self.write(':') + self.write(":") self.body_or_else(node) def visit_With(self, node): self.newline() - self.write('with ') + self.write("with ") self.visit(node.context_expr) if node.optional_vars is not None: - self.write(' as ') + self.write(" as ") self.visit(node.optional_vars) - self.write(':') + self.write(":") self.body(node.body) def visit_Pass(self, node): self.newline() - self.write('pass') + self.write("pass") def visit_Print(self, node): # XXX: python 2.6 only self.newline() - self.write('print ') + self.write("print ") want_comma = False if node.dest is not None: - self.write(' >> ') + self.write(" >> ") self.visit(node.dest) want_comma = True for value in node.values: if want_comma: - self.write(', ') + self.write(", ") self.visit(value) want_comma = True if not node.nl: - self.write(',') + self.write(",") def visit_Delete(self, node): self.newline() - self.write('del ') + self.write("del ") for idx, target in enumerate(node): if idx: - self.write(', ') + self.write(", ") self.visit(target) def visit_TryExcept(self, node): self.newline() - self.write('try:') + self.write("try:") self.body(node.body) for handler in node.handlers: self.visit(handler) def visit_TryFinally(self, node): self.newline() - self.write('try:') + self.write("try:") self.body(node.body) self.newline() - self.write('finally:') + self.write("finally:") self.body(node.finalbody) def visit_Global(self, node): self.newline() - self.write('global ' + ', '.join(node.names)) + self.write("global " + ", ".join(node.names)) def visit_Nonlocal(self, node): self.newline() - self.write('nonlocal ' + ', '.join(node.names)) + self.write("nonlocal " + ", ".join(node.names)) def visit_Return(self, node): self.newline() - self.write('return ') + self.write("return ") self.visit(node.value) def visit_Break(self, node): self.newline() - self.write('break') + self.write("break") def visit_Continue(self, node): self.newline() - self.write('continue') + self.write("continue") def visit_Raise(self, node): # XXX: Python 2.6 / 3.0 compatibility self.newline() - self.write('raise') - if hasattr(node, 'exc') and node.exc is not None: - self.write(' ') + self.write("raise") + if hasattr(node, "exc") and node.exc is not None: + self.write(" ") self.visit(node.exc) if node.cause is not None: - self.write(' from ') + self.write(" from ") self.visit(node.cause) - elif hasattr(node, 'type') and node.type is not None: + elif hasattr(node, "type") and node.type is not None: self.visit(node.type) if node.inst is not None: - self.write(', ') + self.write(", ") self.visit(node.inst) if node.tback is not None: - self.write(', ') + self.write(", ") self.visit(node.tback) # Expressions def visit_Attribute(self, node): self.visit(node.value) - self.write('.' + node.attr) + self.write("." + node.attr) def visit_Call(self, node): want_comma = [] def write_comma(): if want_comma: - self.write(', ') + self.write(", ") else: want_comma.append(True) self.visit(node.func) - self.write('(') + self.write("(") for arg in node.args: write_comma() self.visit(arg) for keyword in node.keywords: write_comma() - self.write(keyword.arg + '=') + self.write(keyword.arg + "=") self.visit(keyword.value) if getattr(node, "starargs", None): write_comma() - self.write('*') + self.write("*") self.visit(node.starargs) if getattr(node, "kwargs", None): write_comma() - self.write('**') + self.write("**") self.visit(node.kwargs) - self.write(')') + self.write(")") def visit_Name(self, node): self.write(node.id) @@ -680,106 +539,111 @@ class SourceGenerator(NodeVisitor): def visit_Num(self, node): self.write(repr(node.n)) + # newly needed in Python 3.8 + def visit_Constant(self, node): + self.write(repr(node.value)) + def visit_Tuple(self, node): - self.write('(') + self.write("(") idx = -1 for idx, item in enumerate(node.elts): if idx: - self.write(', ') + self.write(", ") self.visit(item) - self.write(idx and ')' or ',)') + self.write(idx and ")" or ",)") def sequence_visit(left, right): def visit(self, node): self.write(left) for idx, item in enumerate(node.elts): if idx: - self.write(', ') + self.write(", ") self.visit(item) self.write(right) + return visit - visit_List = sequence_visit('[', ']') - visit_Set = sequence_visit('{', '}') + visit_List = sequence_visit("[", "]") + visit_Set = sequence_visit("{", "}") del sequence_visit def visit_Dict(self, node): - self.write('{') + self.write("{") for idx, (key, value) in enumerate(zip(node.keys, node.values)): if idx: - self.write(', ') + self.write(", ") self.visit(key) - self.write(': ') + self.write(": ") self.visit(value) - self.write('}') + self.write("}") def visit_BinOp(self, node): - self.write('(') + self.write("(") self.visit(node.left) - self.write(' %s ' % BINOP_SYMBOLS[type(node.op)]) + self.write(" %s " % BINOP_SYMBOLS[type(node.op)]) self.visit(node.right) - self.write(')') + self.write(")") def visit_BoolOp(self, node): - self.write('(') + self.write("(") for idx, value in enumerate(node.values): if idx: - self.write(' %s ' % BOOLOP_SYMBOLS[type(node.op)]) + self.write(" %s " % BOOLOP_SYMBOLS[type(node.op)]) self.visit(value) - self.write(')') + self.write(")") def visit_Compare(self, node): - self.write('(') + self.write("(") self.visit(node.left) for op, right in zip(node.ops, node.comparators): - self.write(' %s ' % CMPOP_SYMBOLS[type(op)]) + self.write(" %s " % CMPOP_SYMBOLS[type(op)]) self.visit(right) - self.write(')') + self.write(")") def visit_UnaryOp(self, node): - self.write('(') + self.write("(") op = UNARYOP_SYMBOLS[type(node.op)] self.write(op) - if op == 'not': - self.write(' ') + if op == "not": + self.write(" ") self.visit(node.operand) - self.write(')') + self.write(")") def visit_Subscript(self, node): self.visit(node.value) - self.write('[') + self.write("[") self.visit(node.slice) - self.write(']') + self.write("]") def visit_Slice(self, node): if node.lower is not None: self.visit(node.lower) - self.write(':') + self.write(":") if node.upper is not None: self.visit(node.upper) if node.step is not None: - self.write(':') - if not (isinstance(node.step, Name) and node.step.id == 'None'): + self.write(":") + if not (isinstance(node.step, Name) and node.step.id == "None"): self.visit(node.step) def visit_ExtSlice(self, node): for idx, item in node.dims: if idx: - self.write(', ') + self.write(", ") self.visit(item) def visit_Yield(self, node): - self.write('yield ') + self.write("yield ") self.visit(node.value) def visit_Lambda(self, node): - self.write('lambda ') + self.write("lambda ") self.signature(node.args) - self.write(': ') + self.write(": ") self.visit(node.body) def visit_Ellipsis(self, node): - self.write('Ellipsis') + self.write("Ellipsis") def generator_visit(left, right): def visit(self, node): @@ -788,64 +652,65 @@ class SourceGenerator(NodeVisitor): for comprehension in node.generators: self.visit(comprehension) self.write(right) + return visit - visit_ListComp = generator_visit('[', ']') - visit_GeneratorExp = generator_visit('(', ')') - visit_SetComp = generator_visit('{', '}') + visit_ListComp = generator_visit("[", "]") + visit_GeneratorExp = generator_visit("(", ")") + visit_SetComp = generator_visit("{", "}") del generator_visit def visit_DictComp(self, node): - self.write('{') + self.write("{") self.visit(node.key) - self.write(': ') + self.write(": ") self.visit(node.value) for comprehension in node.generators: self.visit(comprehension) - self.write('}') + self.write("}") def visit_IfExp(self, node): self.visit(node.body) - self.write(' if ') + self.write(" if ") self.visit(node.test) - self.write(' else ') + self.write(" else ") self.visit(node.orelse) def visit_Starred(self, node): - self.write('*') + self.write("*") self.visit(node.value) def visit_Repr(self, node): # XXX: python 2.6 only - self.write('`') + self.write("`") self.visit(node.value) - self.write('`') + self.write("`") # Helper Nodes def visit_alias(self, node): self.write(node.name) if node.asname is not None: - self.write(' as ' + node.asname) + self.write(" as " + node.asname) def visit_comprehension(self, node): - self.write(' for ') + self.write(" for ") self.visit(node.target) - self.write(' in ') + self.write(" in ") self.visit(node.iter) if node.ifs: for if_ in node.ifs: - self.write(' if ') + self.write(" if ") self.visit(if_) def visit_excepthandler(self, node): self.newline() - self.write('except') + self.write("except") if node.type is not None: - self.write(' ') + self.write(" ") self.visit(node.type) if node.name is not None: - self.write(' as ') + self.write(" as ") self.visit(node.name) - self.write(':') + self.write(":") self.body(node.body) diff --git a/server/www/packages/packages-linux/x64/mako/ast.py b/server/www/packages/packages-linux/x64/mako/ast.py index 8d2d150..8f2cf2e 100644 --- a/server/www/packages/packages-linux/x64/mako/ast.py +++ b/server/www/packages/packages-linux/x64/mako/ast.py @@ -1,5 +1,5 @@ # mako/ast.py -# Copyright (C) 2006-2016 the Mako authors and contributors +# Copyright 2006-2019 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -7,9 +7,12 @@ """utilities for analyzing expressions and blocks of Python code, as well as generating Python from AST nodes""" -from mako import exceptions, pyparser, compat import re +from mako import compat +from mako import exceptions +from mako import pyparser + class PythonCode(object): @@ -72,36 +75,39 @@ class PythonFragment(PythonCode): """extends PythonCode to provide identifier lookups in partial control statements - e.g. + e.g.:: + for x in 5: elif y==9: except (MyException, e): - etc. + """ def __init__(self, code, **exception_kwargs): - m = re.match(r'^(\w+)(?:\s+(.*?))?:\s*(#|$)', code.strip(), re.S) + m = re.match(r"^(\w+)(?:\s+(.*?))?:\s*(#|$)", code.strip(), re.S) if not m: raise exceptions.CompileException( - "Fragment '%s' is not a partial control statement" % - code, **exception_kwargs) + "Fragment '%s' is not a partial control statement" % code, + **exception_kwargs + ) if m.group(3): - code = code[:m.start(3)] + code = code[: m.start(3)] (keyword, expr) = m.group(1, 2) - if keyword in ['for', 'if', 'while']: + if keyword in ["for", "if", "while"]: code = code + "pass" - elif keyword == 'try': + elif keyword == "try": code = code + "pass\nexcept:pass" - elif keyword == 'elif' or keyword == 'else': + elif keyword == "elif" or keyword == "else": code = "if False:pass\n" + code + "pass" - elif keyword == 'except': + elif keyword == "except": code = "try:pass\n" + code + "pass" - elif keyword == 'with': + elif keyword == "with": code = code + "pass" else: raise exceptions.CompileException( - "Unsupported control keyword: '%s'" % - keyword, **exception_kwargs) + "Unsupported control keyword: '%s'" % keyword, + **exception_kwargs + ) super(PythonFragment, self).__init__(code, **exception_kwargs) @@ -115,14 +121,17 @@ class FunctionDecl(object): f = pyparser.ParseFunc(self, **exception_kwargs) f.visit(expr) - if not hasattr(self, 'funcname'): + if not hasattr(self, "funcname"): raise exceptions.CompileException( "Code '%s' is not a function declaration" % code, - **exception_kwargs) + **exception_kwargs + ) if not allow_kwargs and self.kwargs: raise exceptions.CompileException( - "'**%s' keyword argument not allowed here" % - self.kwargnames[-1], **exception_kwargs) + "'**%s' keyword argument not allowed here" + % self.kwargnames[-1], + **exception_kwargs + ) def get_argument_expressions(self, as_call=False): """Return the argument declarations of this FunctionDecl as a printable @@ -157,8 +166,10 @@ class FunctionDecl(object): # `def foo(*, a=1, b, c=3)` namedecls.append(name) else: - namedecls.append("%s=%s" % ( - name, pyparser.ExpressionGenerator(default).value())) + namedecls.append( + "%s=%s" + % (name, pyparser.ExpressionGenerator(default).value()) + ) else: namedecls.append(name) @@ -171,8 +182,10 @@ class FunctionDecl(object): namedecls.append(name) else: default = defaults.pop(0) - namedecls.append("%s=%s" % ( - name, pyparser.ExpressionGenerator(default).value())) + namedecls.append( + "%s=%s" + % (name, pyparser.ExpressionGenerator(default).value()) + ) namedecls.reverse() return namedecls @@ -187,5 +200,6 @@ class FunctionArgs(FunctionDecl): """the argument portion of a function declaration""" def __init__(self, code, **kwargs): - super(FunctionArgs, self).__init__("def ANON(%s):pass" % code, - **kwargs) + super(FunctionArgs, self).__init__( + "def ANON(%s):pass" % code, **kwargs + ) diff --git a/server/www/packages/packages-linux/x64/mako/cache.py b/server/www/packages/packages-linux/x64/mako/cache.py index 1af17dd..b68b74f 100644 --- a/server/www/packages/packages-linux/x64/mako/cache.py +++ b/server/www/packages/packages-linux/x64/mako/cache.py @@ -1,10 +1,11 @@ # mako/cache.py -# Copyright (C) 2006-2016 the Mako authors and contributors +# Copyright 2006-2019 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -from mako import compat, util +from mako import compat +from mako import util _cache_plugins = util.PluginLoader("mako.cache") @@ -90,9 +91,8 @@ class Cache(object): return creation_function() return self.impl.get_or_create( - key, - creation_function, - **self._get_cache_kw(kw, context)) + key, creation_function, **self._get_cache_kw(kw, context) + ) def set(self, key, value, **kw): r"""Place a value in the cache. @@ -141,7 +141,7 @@ class Cache(object): template. """ - self.invalidate('render_body', __M_defname='render_body') + self.invalidate("render_body", __M_defname="render_body") def invalidate_def(self, name): """Invalidate the cached content of a particular ``<%def>`` within this @@ -149,7 +149,7 @@ class Cache(object): """ - self.invalidate('render_%s' % name, __M_defname='render_%s' % name) + self.invalidate("render_%s" % name, __M_defname="render_%s" % name) def invalidate_closure(self, name): """Invalidate a nested ``<%def>`` within this template. @@ -165,7 +165,7 @@ class Cache(object): self.invalidate(name, __M_defname=name) def _get_cache_kw(self, kw, context): - defname = kw.pop('__M_defname', None) + defname = kw.pop("__M_defname", None) if not defname: tmpl_kw = self.template.cache_args.copy() tmpl_kw.update(kw) @@ -177,7 +177,7 @@ class Cache(object): self._def_regions[defname] = tmpl_kw if context and self.impl.pass_context: tmpl_kw = tmpl_kw.copy() - tmpl_kw.setdefault('context', context) + tmpl_kw.setdefault("context", context) return tmpl_kw diff --git a/server/www/packages/packages-linux/x64/mako/cmd.py b/server/www/packages/packages-linux/x64/mako/cmd.py index 8db1346..5d52dfb 100644 --- a/server/www/packages/packages-linux/x64/mako/cmd.py +++ b/server/www/packages/packages-linux/x64/mako/cmd.py @@ -1,14 +1,16 @@ # mako/cmd.py -# Copyright (C) 2006-2016 the Mako authors and contributors +# Copyright 2006-2019 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php from argparse import ArgumentParser -from os.path import isfile, dirname +from os.path import dirname +from os.path import isfile import sys -from mako.template import Template -from mako.lookup import TemplateLookup + from mako import exceptions +from mako.lookup import TemplateLookup +from mako.template import Template def varsplit(var): @@ -24,25 +26,41 @@ def _exit(): def cmdline(argv=None): - parser = ArgumentParser("usage: %prog [FILENAME]") + parser = ArgumentParser() parser.add_argument( - "--var", default=[], action="append", - help="variable (can be used multiple times, use name=value)") + "--var", + default=[], + action="append", + help="variable (can be used multiple times, use name=value)", + ) parser.add_argument( - "--template-dir", default=[], action="append", + "--template-dir", + default=[], + action="append", help="Directory to use for template lookup (multiple " "directories may be provided). If not given then if the " "template is read from stdin, the value defaults to be " "the current directory, otherwise it defaults to be the " - "parent directory of the file provided.") - parser.add_argument('input', nargs='?', default='-') + "parent directory of the file provided.", + ) + parser.add_argument( + "--output-encoding", default=None, help="force output encoding" + ) + parser.add_argument("input", nargs="?", default="-") options = parser.parse_args(argv) - if options.input == '-': + + output_encoding = options.output_encoding + + if options.input == "-": lookup_dirs = options.template_dir or ["."] lookup = TemplateLookup(lookup_dirs) try: - template = Template(sys.stdin.read(), lookup=lookup) + template = Template( + sys.stdin.read(), + lookup=lookup, + output_encoding=output_encoding, + ) except: _exit() else: @@ -52,7 +70,11 @@ def cmdline(argv=None): lookup_dirs = options.template_dir or [dirname(filename)] lookup = TemplateLookup(lookup_dirs) try: - template = Template(filename=filename, lookup=lookup) + template = Template( + filename=filename, + lookup=lookup, + output_encoding=output_encoding, + ) except: _exit() diff --git a/server/www/packages/packages-linux/x64/mako/codegen.py b/server/www/packages/packages-linux/x64/mako/codegen.py index d4ecbe8..8f9eef4 100644 --- a/server/www/packages/packages-linux/x64/mako/codegen.py +++ b/server/www/packages/packages-linux/x64/mako/codegen.py @@ -1,5 +1,5 @@ # mako/codegen.py -# Copyright (C) 2006-2016 the Mako authors and contributors +# Copyright 2006-2019 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -7,11 +7,17 @@ """provides functionality for rendering a parsetree constructing into module source code.""" -import time +import json import re -from mako.pygen import PythonPrinter -from mako import util, ast, parsetree, filters, exceptions +import time + +from mako import ast from mako import compat +from mako import exceptions +from mako import filters +from mako import parsetree +from mako import util +from mako.pygen import PythonPrinter MAGIC_NUMBER = 10 @@ -20,22 +26,24 @@ MAGIC_NUMBER = 10 # template and are not accessed via the # context itself TOPLEVEL_DECLARED = set(["UNDEFINED", "STOP_RENDERING"]) -RESERVED_NAMES = set(['context', 'loop']).union(TOPLEVEL_DECLARED) +RESERVED_NAMES = set(["context", "loop"]).union(TOPLEVEL_DECLARED) -def compile(node, - uri, - filename=None, - default_filters=None, - buffer_filters=None, - imports=None, - future_imports=None, - source_encoding=None, - generate_magic_comment=True, - disable_unicode=False, - strict_undefined=False, - enable_loop=True, - reserved_names=frozenset()): +def compile( # noqa + node, + uri, + filename=None, + default_filters=None, + buffer_filters=None, + imports=None, + future_imports=None, + source_encoding=None, + generate_magic_comment=True, + disable_unicode=False, + strict_undefined=False, + enable_loop=True, + reserved_names=frozenset(), +): """Generate module source code given a parsetree node, uri, and optional source filename""" @@ -49,38 +57,43 @@ def compile(node, buf = util.FastEncodingBuffer() printer = PythonPrinter(buf) - _GenerateRenderMethod(printer, - _CompileContext(uri, - filename, - default_filters, - buffer_filters, - imports, - future_imports, - source_encoding, - generate_magic_comment, - disable_unicode, - strict_undefined, - enable_loop, - reserved_names), - node) + _GenerateRenderMethod( + printer, + _CompileContext( + uri, + filename, + default_filters, + buffer_filters, + imports, + future_imports, + source_encoding, + generate_magic_comment, + disable_unicode, + strict_undefined, + enable_loop, + reserved_names, + ), + node, + ) return buf.getvalue() class _CompileContext(object): - - def __init__(self, - uri, - filename, - default_filters, - buffer_filters, - imports, - future_imports, - source_encoding, - generate_magic_comment, - disable_unicode, - strict_undefined, - enable_loop, - reserved_names): + def __init__( + self, + uri, + filename, + default_filters, + buffer_filters, + imports, + future_imports, + source_encoding, + generate_magic_comment, + disable_unicode, + strict_undefined, + enable_loop, + reserved_names, + ): self.uri = uri self.filename = filename self.default_filters = default_filters @@ -113,12 +126,12 @@ class _GenerateRenderMethod(object): name = "render_%s" % node.funcname args = node.get_argument_expressions() filtered = len(node.filter_args.args) > 0 - buffered = eval(node.attributes.get('buffered', 'False')) - cached = eval(node.attributes.get('cached', 'False')) + buffered = eval(node.attributes.get("buffered", "False")) + cached = eval(node.attributes.get("cached", "False")) defs = None pagetag = None if node.is_block and not node.is_anonymous: - args += ['**pageargs'] + args += ["**pageargs"] else: defs = self.write_toplevel() pagetag = self.compiler.pagetag @@ -126,25 +139,23 @@ class _GenerateRenderMethod(object): if pagetag is not None: args = pagetag.body_decl.get_argument_expressions() if not pagetag.body_decl.kwargs: - args += ['**pageargs'] - cached = eval(pagetag.attributes.get('cached', 'False')) + args += ["**pageargs"] + cached = eval(pagetag.attributes.get("cached", "False")) self.compiler.enable_loop = self.compiler.enable_loop or eval( - pagetag.attributes.get( - 'enable_loop', 'False') + pagetag.attributes.get("enable_loop", "False") ) else: - args = ['**pageargs'] + args = ["**pageargs"] cached = False buffered = filtered = False if args is None: - args = ['context'] + args = ["context"] else: - args = [a for a in ['context'] + args] + args = [a for a in ["context"] + args] self.write_render_callable( - pagetag or node, - name, args, - buffered, filtered, cached) + pagetag or node, name, args, buffered, filtered, cached + ) if defs is not None: for node in defs: @@ -154,8 +165,9 @@ class _GenerateRenderMethod(object): self.write_metadata_struct() def write_metadata_struct(self): - self.printer.source_map[self.printer.lineno] = \ - max(self.printer.source_map) + self.printer.source_map[self.printer.lineno] = max( + self.printer.source_map + ) struct = { "filename": self.compiler.filename, "uri": self.compiler.uri, @@ -164,10 +176,9 @@ class _GenerateRenderMethod(object): } self.printer.writelines( '"""', - '__M_BEGIN_METADATA', - compat.json.dumps(struct), - '__M_END_METADATA\n' - '"""' + "__M_BEGIN_METADATA", + json.dumps(struct), + "__M_END_METADATA\n" '"""', ) @property @@ -186,7 +197,6 @@ class _GenerateRenderMethod(object): self.compiler.pagetag = None class FindTopLevel(object): - def visitInheritTag(s, node): inherit.append(node) @@ -214,14 +224,19 @@ class _GenerateRenderMethod(object): module_identifiers.declared = module_ident # module-level names, python code - if self.compiler.generate_magic_comment and \ - self.compiler.source_encoding: - self.printer.writeline("# -*- coding:%s -*-" % - self.compiler.source_encoding) + if ( + self.compiler.generate_magic_comment + and self.compiler.source_encoding + ): + self.printer.writeline( + "# -*- coding:%s -*-" % self.compiler.source_encoding + ) if self.compiler.future_imports: - self.printer.writeline("from __future__ import %s" % - (", ".join(self.compiler.future_imports),)) + self.printer.writeline( + "from __future__ import %s" + % (", ".join(self.compiler.future_imports),) + ) self.printer.writeline("from mako import runtime, filters, cache") self.printer.writeline("UNDEFINED = runtime.UNDEFINED") self.printer.writeline("STOP_RENDERING = runtime.STOP_RENDERING") @@ -231,36 +246,41 @@ class _GenerateRenderMethod(object): self.printer.writeline("_modified_time = %r" % time.time()) self.printer.writeline("_enable_loop = %r" % self.compiler.enable_loop) self.printer.writeline( - "_template_filename = %r" % self.compiler.filename) + "_template_filename = %r" % self.compiler.filename + ) self.printer.writeline("_template_uri = %r" % self.compiler.uri) self.printer.writeline( - "_source_encoding = %r" % self.compiler.source_encoding) + "_source_encoding = %r" % self.compiler.source_encoding + ) if self.compiler.imports: - buf = '' + buf = "" for imp in self.compiler.imports: buf += imp + "\n" self.printer.writeline(imp) impcode = ast.PythonCode( buf, - source='', lineno=0, + source="", + lineno=0, pos=0, - filename='template defined imports') + filename="template defined imports", + ) else: impcode = None main_identifiers = module_identifiers.branch(self.node) - module_identifiers.topleveldefs = \ - module_identifiers.topleveldefs.\ - union(main_identifiers.topleveldefs) + mit = module_identifiers.topleveldefs + module_identifiers.topleveldefs = mit.union( + main_identifiers.topleveldefs + ) module_identifiers.declared.update(TOPLEVEL_DECLARED) if impcode: module_identifiers.declared.update(impcode.declared_identifiers) self.compiler.identifiers = module_identifiers - self.printer.writeline("_exports = %r" % - [n.name for n in - main_identifiers.topleveldefs.values()] - ) + self.printer.writeline( + "_exports = %r" + % [n.name for n in main_identifiers.topleveldefs.values()] + ) self.printer.write_blanks(2) if len(module_code): @@ -274,8 +294,9 @@ class _GenerateRenderMethod(object): return list(main_identifiers.topleveldefs.values()) - def write_render_callable(self, node, name, args, buffered, filtered, - cached): + def write_render_callable( + self, node, name, args, buffered, filtered, cached + ): """write a top-level render callable. this could be the main render() method or that of a top-level def.""" @@ -284,32 +305,38 @@ class _GenerateRenderMethod(object): decorator = node.decorator if decorator: self.printer.writeline( - "@runtime._decorate_toplevel(%s)" % decorator) + "@runtime._decorate_toplevel(%s)" % decorator + ) self.printer.start_source(node.lineno) self.printer.writelines( - "def %s(%s):" % (name, ','.join(args)), + "def %s(%s):" % (name, ",".join(args)), # push new frame, assign current frame to __M_caller "__M_caller = context.caller_stack._push_frame()", - "try:" + "try:", ) if buffered or filtered or cached: self.printer.writeline("context._push_buffer()") self.identifier_stack.append( - self.compiler.identifiers.branch(self.node)) - if (not self.in_def or self.node.is_block) and '**pageargs' in args: - self.identifier_stack[-1].argument_declared.add('pageargs') + self.compiler.identifiers.branch(self.node) + ) + if (not self.in_def or self.node.is_block) and "**pageargs" in args: + self.identifier_stack[-1].argument_declared.add("pageargs") if not self.in_def and ( - len(self.identifiers.locally_assigned) > 0 or - len(self.identifiers.argument_declared) > 0 + len(self.identifiers.locally_assigned) > 0 + or len(self.identifiers.argument_declared) > 0 ): - self.printer.writeline("__M_locals = __M_dict_builtin(%s)" % - ','.join([ - "%s=%s" % (x, x) for x in - self.identifiers.argument_declared - ])) + self.printer.writeline( + "__M_locals = __M_dict_builtin(%s)" + % ",".join( + [ + "%s=%s" % (x, x) + for x in self.identifiers.argument_declared + ] + ) + ) self.write_variable_declares(self.identifiers, toplevel=True) @@ -321,16 +348,14 @@ class _GenerateRenderMethod(object): self.printer.write_blanks(2) if cached: self.write_cache_decorator( - node, name, - args, buffered, - self.identifiers, toplevel=True) + node, name, args, buffered, self.identifiers, toplevel=True + ) def write_module_code(self, module_code): """write module-level template code, i.e. that which is enclosed in <%! %> tags in the template.""" for n in module_code: - self.printer.start_source(n.lineno) - self.printer.write_indented_block(n.text) + self.printer.write_indented_block(n.text, starting_lineno=n.lineno) def write_inherit(self, node): """write the module-level inheritance-determination callable.""" @@ -338,9 +363,9 @@ class _GenerateRenderMethod(object): self.printer.writelines( "def _mako_inherit(template, context):", "_mako_generate_namespaces(context)", - "return runtime._inherit_from(context, %s, _template_uri)" % - (node.parsed_attributes['file']), - None + "return runtime._inherit_from(context, %s, _template_uri)" + % (node.parsed_attributes["file"]), + None, ) def write_namespaces(self, namespaces): @@ -352,12 +377,13 @@ class _GenerateRenderMethod(object): "except KeyError:", "_mako_generate_namespaces(context)", "return context.namespaces[(__name__, name)]", - None, None + None, + None, ) self.printer.writeline("def _mako_generate_namespaces(context):") for node in namespaces.values(): - if 'import' in node.attributes: + if "import" in node.attributes: self.compiler.has_ns_imports = True self.printer.start_source(node.lineno) if len(node.nodes): @@ -367,7 +393,6 @@ class _GenerateRenderMethod(object): self.in_def = True class NSDefVisitor(object): - def visitDefTag(s, node): s.visitDefOrBase(node) @@ -383,56 +408,54 @@ class _GenerateRenderMethod(object): ) self.write_inline_def(node, identifiers, nested=False) export.append(node.funcname) + vis = NSDefVisitor() for n in node.nodes: n.accept_visitor(vis) - self.printer.writeline("return [%s]" % (','.join(export))) + self.printer.writeline("return [%s]" % (",".join(export))) self.printer.writeline(None) self.in_def = False callable_name = "make_namespace()" else: callable_name = "None" - if 'file' in node.parsed_attributes: + if "file" in node.parsed_attributes: self.printer.writeline( "ns = runtime.TemplateNamespace(%r," " context._clean_inheritance_tokens()," " templateuri=%s, callables=%s, " - " calling_uri=_template_uri)" % - ( + " calling_uri=_template_uri)" + % ( node.name, - node.parsed_attributes.get('file', 'None'), + node.parsed_attributes.get("file", "None"), callable_name, ) ) - elif 'module' in node.parsed_attributes: + elif "module" in node.parsed_attributes: self.printer.writeline( "ns = runtime.ModuleNamespace(%r," " context._clean_inheritance_tokens()," " callables=%s, calling_uri=_template_uri," - " module=%s)" % - ( + " module=%s)" + % ( node.name, callable_name, - node.parsed_attributes.get( - 'module', 'None') + node.parsed_attributes.get("module", "None"), ) ) else: self.printer.writeline( "ns = runtime.Namespace(%r," " context._clean_inheritance_tokens()," - " callables=%s, calling_uri=_template_uri)" % - ( - node.name, - callable_name, - ) + " callables=%s, calling_uri=_template_uri)" + % (node.name, callable_name) ) - if eval(node.attributes.get('inheritable', "False")): + if eval(node.attributes.get("inheritable", "False")): self.printer.writeline("context['self'].%s = ns" % (node.name)) self.printer.writeline( - "context.namespaces[(__name__, %s)] = ns" % repr(node.name)) + "context.namespaces[(__name__, %s)] = ns" % repr(node.name) + ) self.printer.write_blanks(1) if not len(namespaces): self.printer.writeline("pass") @@ -468,7 +491,8 @@ class _GenerateRenderMethod(object): # write closure functions for closures that we define # right here to_write = to_write.union( - [c.funcname for c in identifiers.closuredefs.values()]) + [c.funcname for c in identifiers.closuredefs.values()] + ) # remove identifiers that are declared in the argument # signature of the callable @@ -492,23 +516,22 @@ class _GenerateRenderMethod(object): if limit is not None: to_write = to_write.intersection(limit) - if toplevel and getattr(self.compiler, 'has_ns_imports', False): + if toplevel and getattr(self.compiler, "has_ns_imports", False): self.printer.writeline("_import_ns = {}") self.compiler.has_imports = True for ident, ns in self.compiler.namespaces.items(): - if 'import' in ns.attributes: + if "import" in ns.attributes: self.printer.writeline( "_mako_get_namespace(context, %r)." - "_populate(_import_ns, %r)" % - ( + "_populate(_import_ns, %r)" + % ( ident, - re.split(r'\s*,\s*', ns.attributes['import']) - )) + re.split(r"\s*,\s*", ns.attributes["import"]), + ) + ) if has_loop: - self.printer.writeline( - 'loop = __M_loop = runtime.LoopStack()' - ) + self.printer.writeline("loop = __M_loop = runtime.LoopStack()") for ident in to_write: if ident in comp_idents: @@ -526,37 +549,36 @@ class _GenerateRenderMethod(object): elif ident in self.compiler.namespaces: self.printer.writeline( - "%s = _mako_get_namespace(context, %r)" % - (ident, ident) + "%s = _mako_get_namespace(context, %r)" % (ident, ident) ) else: - if getattr(self.compiler, 'has_ns_imports', False): + if getattr(self.compiler, "has_ns_imports", False): if self.compiler.strict_undefined: self.printer.writelines( - "%s = _import_ns.get(%r, UNDEFINED)" % - (ident, ident), + "%s = _import_ns.get(%r, UNDEFINED)" + % (ident, ident), "if %s is UNDEFINED:" % ident, "try:", "%s = context[%r]" % (ident, ident), "except KeyError:", - "raise NameError(\"'%s' is not defined\")" % - ident, - None, None + "raise NameError(\"'%s' is not defined\")" % ident, + None, + None, ) else: self.printer.writeline( "%s = _import_ns.get" - "(%r, context.get(%r, UNDEFINED))" % - (ident, ident, ident)) + "(%r, context.get(%r, UNDEFINED))" + % (ident, ident, ident) + ) else: if self.compiler.strict_undefined: self.printer.writelines( "try:", "%s = context[%r]" % (ident, ident), "except KeyError:", - "raise NameError(\"'%s' is not defined\")" % - ident, - None + "raise NameError(\"'%s' is not defined\")" % ident, + None, ) else: self.printer.writeline( @@ -572,14 +594,16 @@ class _GenerateRenderMethod(object): nameargs = node.get_argument_expressions(as_call=True) if not self.in_def and ( - len(self.identifiers.locally_assigned) > 0 or - len(self.identifiers.argument_declared) > 0): - nameargs.insert(0, 'context._locals(__M_locals)') + len(self.identifiers.locally_assigned) > 0 + or len(self.identifiers.argument_declared) > 0 + ): + nameargs.insert(0, "context._locals(__M_locals)") else: - nameargs.insert(0, 'context') + nameargs.insert(0, "context") self.printer.writeline("def %s(%s):" % (funcname, ",".join(namedecls))) self.printer.writeline( - "return render_%s(%s)" % (funcname, ",".join(nameargs))) + "return render_%s(%s)" % (funcname, ",".join(nameargs)) + ) self.printer.writeline(None) def write_inline_def(self, node, identifiers, nested): @@ -590,21 +614,21 @@ class _GenerateRenderMethod(object): decorator = node.decorator if decorator: self.printer.writeline( - "@runtime._decorate_inline(context, %s)" % decorator) + "@runtime._decorate_inline(context, %s)" % decorator + ) self.printer.writeline( - "def %s(%s):" % (node.funcname, ",".join(namedecls))) + "def %s(%s):" % (node.funcname, ",".join(namedecls)) + ) filtered = len(node.filter_args.args) > 0 - buffered = eval(node.attributes.get('buffered', 'False')) - cached = eval(node.attributes.get('cached', 'False')) + buffered = eval(node.attributes.get("buffered", "False")) + cached = eval(node.attributes.get("cached", "False")) self.printer.writelines( # push new frame, assign current frame to __M_caller "__M_caller = context.caller_stack._push_frame()", - "try:" + "try:", ) if buffered or filtered or cached: - self.printer.writelines( - "context._push_buffer()", - ) + self.printer.writelines("context._push_buffer()") identifiers = identifiers.branch(node, nested=nested) @@ -618,12 +642,19 @@ class _GenerateRenderMethod(object): self.write_def_finish(node, buffered, filtered, cached) self.printer.writeline(None) if cached: - self.write_cache_decorator(node, node.funcname, - namedecls, False, identifiers, - inline=True, toplevel=False) + self.write_cache_decorator( + node, + node.funcname, + namedecls, + False, + identifiers, + inline=True, + toplevel=False, + ) - def write_def_finish(self, node, buffered, filtered, cached, - callstack=True): + def write_def_finish( + self, node, buffered, filtered, cached, callstack=True + ): """write the end section of a rendering function, either outermost or inline. @@ -636,9 +667,7 @@ class _GenerateRenderMethod(object): self.printer.writeline("return ''") if callstack: self.printer.writelines( - "finally:", - "context.caller_stack._pop_frame()", - None + "finally:", "context.caller_stack._pop_frame()", None ) if buffered or filtered or cached: @@ -648,13 +677,12 @@ class _GenerateRenderMethod(object): # implemenation might be using a context with no # extra buffers self.printer.writelines( - "finally:", - "__M_buf = context._pop_buffer()" + "finally:", "__M_buf = context._pop_buffer()" ) else: self.printer.writelines( "finally:", - "__M_buf, __M_writer = context._pop_buffer_and_writer()" + "__M_buf, __M_writer = context._pop_buffer_and_writer()", ) if callstack: @@ -662,89 +690,100 @@ class _GenerateRenderMethod(object): s = "__M_buf.getvalue()" if filtered: - s = self.create_filter_callable(node.filter_args.args, s, - False) + s = self.create_filter_callable( + node.filter_args.args, s, False + ) self.printer.writeline(None) if buffered and not cached: - s = self.create_filter_callable(self.compiler.buffer_filters, - s, False) + s = self.create_filter_callable( + self.compiler.buffer_filters, s, False + ) if buffered or cached: self.printer.writeline("return %s" % s) else: - self.printer.writelines( - "__M_writer(%s)" % s, - "return ''" - ) + self.printer.writelines("__M_writer(%s)" % s, "return ''") - def write_cache_decorator(self, node_or_pagetag, name, - args, buffered, identifiers, - inline=False, toplevel=False): + def write_cache_decorator( + self, + node_or_pagetag, + name, + args, + buffered, + identifiers, + inline=False, + toplevel=False, + ): """write a post-function decorator to replace a rendering callable with a cached version of itself.""" self.printer.writeline("__M_%s = %s" % (name, name)) - cachekey = node_or_pagetag.parsed_attributes.get('cache_key', - repr(name)) + cachekey = node_or_pagetag.parsed_attributes.get( + "cache_key", repr(name) + ) cache_args = {} if self.compiler.pagetag is not None: cache_args.update( - ( - pa[6:], - self.compiler.pagetag.parsed_attributes[pa] - ) + (pa[6:], self.compiler.pagetag.parsed_attributes[pa]) for pa in self.compiler.pagetag.parsed_attributes - if pa.startswith('cache_') and pa != 'cache_key' + if pa.startswith("cache_") and pa != "cache_key" ) cache_args.update( - ( - pa[6:], - node_or_pagetag.parsed_attributes[pa] - ) for pa in node_or_pagetag.parsed_attributes - if pa.startswith('cache_') and pa != 'cache_key' + (pa[6:], node_or_pagetag.parsed_attributes[pa]) + for pa in node_or_pagetag.parsed_attributes + if pa.startswith("cache_") and pa != "cache_key" ) - if 'timeout' in cache_args: - cache_args['timeout'] = int(eval(cache_args['timeout'])) + if "timeout" in cache_args: + cache_args["timeout"] = int(eval(cache_args["timeout"])) - self.printer.writeline("def %s(%s):" % (name, ','.join(args))) + self.printer.writeline("def %s(%s):" % (name, ",".join(args))) # form "arg1, arg2, arg3=arg3, arg4=arg4", etc. pass_args = [ - "%s=%s" % ((a.split('=')[0],) * 2) if '=' in a else a - for a in args + "%s=%s" % ((a.split("=")[0],) * 2) if "=" in a else a for a in args ] self.write_variable_declares( identifiers, toplevel=toplevel, - limit=node_or_pagetag.undeclared_identifiers() + limit=node_or_pagetag.undeclared_identifiers(), ) if buffered: - s = "context.get('local')."\ - "cache._ctx_get_or_create("\ - "%s, lambda:__M_%s(%s), context, %s__M_defname=%r)" % ( - cachekey, name, ','.join(pass_args), - ''.join(["%s=%s, " % (k, v) - for k, v in cache_args.items()]), - name + s = ( + "context.get('local')." + "cache._ctx_get_or_create(" + "%s, lambda:__M_%s(%s), context, %s__M_defname=%r)" + % ( + cachekey, + name, + ",".join(pass_args), + "".join( + ["%s=%s, " % (k, v) for k, v in cache_args.items()] + ), + name, ) + ) # apply buffer_filters - s = self.create_filter_callable(self.compiler.buffer_filters, s, - False) + s = self.create_filter_callable( + self.compiler.buffer_filters, s, False + ) self.printer.writelines("return " + s, None) else: self.printer.writelines( "__M_writer(context.get('local')." "cache._ctx_get_or_create(" - "%s, lambda:__M_%s(%s), context, %s__M_defname=%r))" % - ( - cachekey, name, ','.join(pass_args), - ''.join(["%s=%s, " % (k, v) - for k, v in cache_args.items()]), + "%s, lambda:__M_%s(%s), context, %s__M_defname=%r))" + % ( + cachekey, + name, + ",".join(pass_args), + "".join( + ["%s=%s, " % (k, v) for k, v in cache_args.items()] + ), name, ), "return ''", - None + None, ) def create_filter_callable(self, args, target, is_expression): @@ -753,24 +792,24 @@ class _GenerateRenderMethod(object): 'default' filter aliases as needed.""" def locate_encode(name): - if re.match(r'decode\..+', name): + if re.match(r"decode\..+", name): return "filters." + name elif self.compiler.disable_unicode: return filters.NON_UNICODE_ESCAPES.get(name, name) else: return filters.DEFAULT_ESCAPES.get(name, name) - if 'n' not in args: + if "n" not in args: if is_expression: if self.compiler.pagetag: args = self.compiler.pagetag.filter_args.args + args - if self.compiler.default_filters: + if self.compiler.default_filters and "n" not in args: args = self.compiler.default_filters + args for e in args: # if filter given as a function, get just the identifier portion - if e == 'n': + if e == "n": continue - m = re.match(r'(.+?)(\(.*\))', e) + m = re.match(r"(.+?)(\(.*\))", e) if m: ident, fargs = m.group(1, 2) f = locate_encode(ident) @@ -783,15 +822,18 @@ class _GenerateRenderMethod(object): def visitExpression(self, node): self.printer.start_source(node.lineno) - if len(node.escapes) or \ - ( - self.compiler.pagetag is not None and - len(self.compiler.pagetag.filter_args.args) - ) or \ - len(self.compiler.default_filters): + if ( + len(node.escapes) + or ( + self.compiler.pagetag is not None + and len(self.compiler.pagetag.filter_args.args) + ) + or len(self.compiler.default_filters) + ): - s = self.create_filter_callable(node.escapes_code.args, - "%s" % node.text, True) + s = self.create_filter_callable( + node.escapes_code.args, "%s" % node.text, True + ) self.printer.writeline("__M_writer(%s)" % s) else: self.printer.writeline("__M_writer(%s)" % node.text) @@ -800,12 +842,12 @@ class _GenerateRenderMethod(object): if node.isend: self.printer.writeline(None) if node.has_loop_context: - self.printer.writeline('finally:') + self.printer.writeline("finally:") self.printer.writeline("loop = __M_loop._exit()") self.printer.writeline(None) else: self.printer.start_source(node.lineno) - if self.compiler.enable_loop and node.keyword == 'for': + if self.compiler.enable_loop and node.keyword == "for": text = mangle_mako_loop(node, self.printer) else: text = node.text @@ -817,12 +859,16 @@ class _GenerateRenderMethod(object): # and end control lines, and # 3) any control line with no content other than comments if not children or ( - compat.all(isinstance(c, (parsetree.Comment, - parsetree.ControlLine)) - for c in children) and - compat.all((node.is_ternary(c.keyword) or c.isend) - for c in children - if isinstance(c, parsetree.ControlLine))): + compat.all( + isinstance(c, (parsetree.Comment, parsetree.ControlLine)) + for c in children + ) + and compat.all( + (node.is_ternary(c.keyword) or c.isend) + for c in children + if isinstance(c, parsetree.ControlLine) + ) + ): self.printer.writeline("pass") def visitText(self, node): @@ -833,8 +879,7 @@ class _GenerateRenderMethod(object): filtered = len(node.filter_args.args) > 0 if filtered: self.printer.writelines( - "__M_writer = context._push_writer()", - "try:", + "__M_writer = context._push_writer()", "try:" ) for n in node.nodes: n.accept_visitor(self) @@ -842,18 +887,18 @@ class _GenerateRenderMethod(object): self.printer.writelines( "finally:", "__M_buf, __M_writer = context._pop_buffer_and_writer()", - "__M_writer(%s)" % - self.create_filter_callable( - node.filter_args.args, - "__M_buf.getvalue()", - False), - None + "__M_writer(%s)" + % self.create_filter_callable( + node.filter_args.args, "__M_buf.getvalue()", False + ), + None, ) def visitCode(self, node): if not node.ismodule: - self.printer.start_source(node.lineno) - self.printer.write_indented_block(node.text) + self.printer.write_indented_block( + node.text, starting_lineno=node.lineno + ) if not self.in_def and len(self.identifiers.locally_assigned) > 0: # if we are the "template" def, fudge locally @@ -861,24 +906,28 @@ class _GenerateRenderMethod(object): # which is used for def calls within the same template, # to simulate "enclosing scope" self.printer.writeline( - '__M_locals_builtin_stored = __M_locals_builtin()') + "__M_locals_builtin_stored = __M_locals_builtin()" + ) self.printer.writeline( - '__M_locals.update(__M_dict_builtin([(__M_key,' - ' __M_locals_builtin_stored[__M_key]) for __M_key in' - ' [%s] if __M_key in __M_locals_builtin_stored]))' % - ','.join([repr(x) for x in node.declared_identifiers()])) + "__M_locals.update(__M_dict_builtin([(__M_key," + " __M_locals_builtin_stored[__M_key]) for __M_key in" + " [%s] if __M_key in __M_locals_builtin_stored]))" + % ",".join([repr(x) for x in node.declared_identifiers()]) + ) def visitIncludeTag(self, node): self.printer.start_source(node.lineno) - args = node.attributes.get('args') + args = node.attributes.get("args") if args: self.printer.writeline( - "runtime._include_file(context, %s, _template_uri, %s)" % - (node.parsed_attributes['file'], args)) + "runtime._include_file(context, %s, _template_uri, %s)" + % (node.parsed_attributes["file"], args) + ) else: self.printer.writeline( - "runtime._include_file(context, %s, _template_uri)" % - (node.parsed_attributes['file'])) + "runtime._include_file(context, %s, _template_uri)" + % (node.parsed_attributes["file"]) + ) def visitNamespaceTag(self, node): pass @@ -891,13 +940,14 @@ class _GenerateRenderMethod(object): self.printer.writeline("%s()" % node.funcname) else: nameargs = node.get_argument_expressions(as_call=True) - nameargs += ['**pageargs'] + nameargs += ["**pageargs"] self.printer.writeline( "if 'parent' not in context._data or " - "not hasattr(context._data['parent'], '%s'):" - % node.funcname) + "not hasattr(context._data['parent'], '%s'):" % node.funcname + ) self.printer.writeline( - "context['self'].%s(%s)" % (node.funcname, ",".join(nameargs))) + "context['self'].%s(%s)" % (node.funcname, ",".join(nameargs)) + ) self.printer.writeline("\n") def visitCallNamespaceTag(self, node): @@ -908,19 +958,18 @@ class _GenerateRenderMethod(object): def visitCallTag(self, node): self.printer.writeline("def ccall(caller):") - export = ['body'] + export = ["body"] callable_identifiers = self.identifiers.branch(node, nested=True) body_identifiers = callable_identifiers.branch(node, nested=False) # we want the 'caller' passed to ccall to be used # for the body() function, but for other non-body() # <%def>s within <%call> we want the current caller # off the call stack (if any) - body_identifiers.add_declared('caller') + body_identifiers.add_declared("caller") self.identifier_stack.append(body_identifiers) class DefVisitor(object): - def visitDefTag(s, node): s.visitDefOrBase(node) @@ -942,16 +991,13 @@ class _GenerateRenderMethod(object): self.identifier_stack.pop() bodyargs = node.body_decl.get_argument_expressions() - self.printer.writeline("def body(%s):" % ','.join(bodyargs)) + self.printer.writeline("def body(%s):" % ",".join(bodyargs)) # TODO: figure out best way to specify # buffering/nonbuffering (at call time would be better) buffered = False if buffered: - self.printer.writelines( - "context._push_buffer()", - "try:" - ) + self.printer.writelines("context._push_buffer()", "try:") self.write_variable_declares(body_identifiers) self.identifier_stack.append(body_identifiers) @@ -960,25 +1006,22 @@ class _GenerateRenderMethod(object): self.identifier_stack.pop() self.write_def_finish(node, buffered, False, False, callstack=False) - self.printer.writelines( - None, - "return [%s]" % (','.join(export)), - None - ) + self.printer.writelines(None, "return [%s]" % (",".join(export)), None) self.printer.writelines( # push on caller for nested call "context.caller_stack.nextcaller = " "runtime.Namespace('caller', context, " "callables=ccall(__M_caller))", - "try:") + "try:", + ) self.printer.start_source(node.lineno) self.printer.writelines( - "__M_writer(%s)" % self.create_filter_callable( - [], node.expression, True), + "__M_writer(%s)" + % self.create_filter_callable([], node.expression, True), "finally:", "context.caller_stack.nextcaller = None", - None + None, ) @@ -996,10 +1039,12 @@ class _Identifiers(object): else: # things that have already been declared # in an enclosing namespace (i.e. names we can just use) - self.declared = set(parent.declared).\ - union([c.name for c in parent.closuredefs.values()]).\ - union(parent.locally_declared).\ - union(parent.argument_declared) + self.declared = ( + set(parent.declared) + .union([c.name for c in parent.closuredefs.values()]) + .union(parent.locally_declared) + .union(parent.argument_declared) + ) # if these identifiers correspond to a "nested" # scope, it means whatever the parent identifiers @@ -1043,11 +1088,13 @@ class _Identifiers(object): node.accept_visitor(self) illegal_names = self.compiler.reserved_names.intersection( - self.locally_declared) + self.locally_declared + ) if illegal_names: raise exceptions.NameConflictError( - "Reserved words declared in template: %s" % - ", ".join(illegal_names)) + "Reserved words declared in template: %s" + % ", ".join(illegal_names) + ) def branch(self, node, **kwargs): """create a new Identifiers for a new Node, with @@ -1060,24 +1107,28 @@ class _Identifiers(object): return set(self.topleveldefs.union(self.closuredefs).values()) def __repr__(self): - return "Identifiers(declared=%r, locally_declared=%r, "\ - "undeclared=%r, topleveldefs=%r, closuredefs=%r, "\ - "argumentdeclared=%r)" %\ - ( + return ( + "Identifiers(declared=%r, locally_declared=%r, " + "undeclared=%r, topleveldefs=%r, closuredefs=%r, " + "argumentdeclared=%r)" + % ( list(self.declared), list(self.locally_declared), list(self.undeclared), [c.name for c in self.topleveldefs.values()], [c.name for c in self.closuredefs.values()], - self.argument_declared) + self.argument_declared, + ) + ) def check_declared(self, node): """update the state of this Identifiers with the undeclared and declared identifiers of the given node.""" for ident in node.undeclared_identifiers(): - if ident != 'context' and\ - ident not in self.declared.union(self.locally_declared): + if ident != "context" and ident not in self.declared.union( + self.locally_declared + ): self.undeclared.add(ident) for ident in node.declared_identifiers(): self.locally_declared.add(ident) @@ -1097,7 +1148,8 @@ class _Identifiers(object): if not node.ismodule: self.check_declared(node) self.locally_assigned = self.locally_assigned.union( - node.declared_identifiers()) + node.declared_identifiers() + ) def visitNamespaceTag(self, node): # only traverse into the sub-elements of a @@ -1110,13 +1162,16 @@ class _Identifiers(object): def _check_name_exists(self, collection, node): existing = collection.get(node.funcname) collection[node.funcname] = node - if existing is not None and \ - existing is not node and \ - (node.is_block or existing.is_block): + if ( + existing is not None + and existing is not node + and (node.is_block or existing.is_block) + ): raise exceptions.CompileException( "%%def or %%block named '%s' already " - "exists in this template." % - node.funcname, **node.exception_kwargs) + "exists in this template." % node.funcname, + **node.exception_kwargs + ) def visitDefTag(self, node): if node.is_root() and not node.is_anonymous: @@ -1125,8 +1180,9 @@ class _Identifiers(object): self._check_name_exists(self.closuredefs, node) for ident in node.undeclared_identifiers(): - if ident != 'context' and \ - ident not in self.declared.union(self.locally_declared): + if ident != "context" and ident not in self.declared.union( + self.locally_declared + ): self.undeclared.add(ident) # visit defs only one level deep @@ -1143,16 +1199,22 @@ class _Identifiers(object): if isinstance(self.node, parsetree.DefTag): raise exceptions.CompileException( "Named block '%s' not allowed inside of def '%s'" - % (node.name, self.node.name), **node.exception_kwargs) - elif isinstance(self.node, - (parsetree.CallTag, parsetree.CallNamespaceTag)): + % (node.name, self.node.name), + **node.exception_kwargs + ) + elif isinstance( + self.node, (parsetree.CallTag, parsetree.CallNamespaceTag) + ): raise exceptions.CompileException( "Named block '%s' not allowed inside of <%%call> tag" - % (node.name, ), **node.exception_kwargs) + % (node.name,), + **node.exception_kwargs + ) for ident in node.undeclared_identifiers(): - if ident != 'context' and \ - ident not in self.declared.union(self.locally_declared): + if ident != "context" and ident not in self.declared.union( + self.locally_declared + ): self.undeclared.add(ident) if not node.is_anonymous: @@ -1167,8 +1229,9 @@ class _Identifiers(object): def visitTextTag(self, node): for ident in node.undeclared_identifiers(): - if ident != 'context' and \ - ident not in self.declared.union(self.locally_declared): + if ident != "context" and ident not in self.declared.union( + self.locally_declared + ): self.undeclared.add(ident) def visitIncludeTag(self, node): @@ -1185,9 +1248,9 @@ class _Identifiers(object): def visitCallTag(self, node): if node is self.node: for ident in node.undeclared_identifiers(): - if ident != 'context' and \ - ident not in self.declared.union( - self.locally_declared): + if ident != "context" and ident not in self.declared.union( + self.locally_declared + ): self.undeclared.add(ident) for ident in node.declared_identifiers(): self.argument_declared.add(ident) @@ -1195,15 +1258,15 @@ class _Identifiers(object): n.accept_visitor(self) else: for ident in node.undeclared_identifiers(): - if ident != 'context' and \ - ident not in self.declared.union( - self.locally_declared): + if ident != "context" and ident not in self.declared.union( + self.locally_declared + ): self.undeclared.add(ident) _FOR_LOOP = re.compile( - r'^for\s+((?:\(?)\s*[A-Za-z_][A-Za-z_0-9]*' - r'(?:\s*,\s*(?:[A-Za-z_][A-Za-z0-9_]*),??)*\s*(?:\)?))\s+in\s+(.*):' + r"^for\s+((?:\(?)\s*[A-Za-z_][A-Za-z_0-9]*" + r"(?:\s*,\s*(?:[A-Za-z_][A-Za-z0-9_]*),??)*\s*(?:\)?))\s+in\s+(.*):" ) @@ -1218,11 +1281,11 @@ def mangle_mako_loop(node, printer): match = _FOR_LOOP.match(node.text) if match: printer.writelines( - 'loop = __M_loop._enter(%s)' % match.group(2), - 'try:' + "loop = __M_loop._enter(%s)" % match.group(2), + "try:" # 'with __M_loop(%s) as loop:' % match.group(2) ) - text = 'for %s in loop:' % match.group(1) + text = "for %s in loop:" % match.group(1) else: raise SyntaxError("Couldn't apply loop context: %s" % node.text) else: @@ -1239,7 +1302,7 @@ class LoopVariable(object): self.detected = False def _loop_reference_detected(self, node): - if 'loop' in node.undeclared_identifiers(): + if "loop" in node.undeclared_identifiers(): self.detected = True else: for n in node.get_children(): diff --git a/server/www/packages/packages-linux/x64/mako/compat.py b/server/www/packages/packages-linux/x64/mako/compat.py index a2ab243..4460fde 100644 --- a/server/www/packages/packages-linux/x64/mako/compat.py +++ b/server/www/packages/packages-linux/x64/mako/compat.py @@ -1,34 +1,52 @@ +# mako/compat.py +# Copyright 2006-2019 the Mako authors and contributors +# +# This module is part of Mako and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +import collections +import inspect import sys -import time py3k = sys.version_info >= (3, 0) -py33 = sys.version_info >= (3, 3) py2k = sys.version_info < (3,) -py26 = sys.version_info >= (2, 6) py27 = sys.version_info >= (2, 7) -jython = sys.platform.startswith('java') -win32 = sys.platform.startswith('win') -pypy = hasattr(sys, 'pypy_version_info') +jython = sys.platform.startswith("java") +win32 = sys.platform.startswith("win") +pypy = hasattr(sys, "pypy_version_info") -if py3k: - # create a "getargspec" from getfullargspec(), which is not deprecated - # in Py3K; getargspec() has started to emit warnings as of Py3.5. - # As of Py3.4, now they are trying to move from getfullargspec() - # to "signature()", but getfullargspec() is not deprecated, so stick - # with that for now. +ArgSpec = collections.namedtuple( + "ArgSpec", ["args", "varargs", "keywords", "defaults"] +) - import collections - ArgSpec = collections.namedtuple( - "ArgSpec", - ["args", "varargs", "keywords", "defaults"]) - from inspect import getfullargspec as inspect_getfullargspec - def inspect_getargspec(func): - return ArgSpec( - *inspect_getfullargspec(func)[0:4] - ) -else: - from inspect import getargspec as inspect_getargspec # noqa +def inspect_getargspec(func): + """getargspec based on fully vendored getfullargspec from Python 3.3.""" + + if inspect.ismethod(func): + func = func.__func__ + if not inspect.isfunction(func): + raise TypeError("{!r} is not a Python function".format(func)) + + co = func.__code__ + if not inspect.iscode(co): + raise TypeError("{!r} is not a code object".format(co)) + + nargs = co.co_argcount + names = co.co_varnames + nkwargs = co.co_kwonlyargcount if py3k else 0 + args = list(names[:nargs]) + + nargs += nkwargs + varargs = None + if co.co_flags & inspect.CO_VARARGS: + varargs = co.co_varnames[nargs] + nargs = nargs + 1 + varkw = None + if co.co_flags & inspect.CO_VARKEYWORDS: + varkw = co.co_varnames[nargs] + + return ArgSpec(args, varargs, varkw, func.__defaults__) if py3k: @@ -36,7 +54,8 @@ if py3k: import builtins as compat_builtins from urllib.parse import quote_plus, unquote_plus from html.entities import codepoint2name, name2codepoint - string_types = str, + + string_types = (str,) binary_type = bytes text_type = str @@ -51,8 +70,10 @@ if py3k: def octal(lit): return eval("0o" + lit) + else: import __builtin__ as compat_builtins # noqa + try: from cStringIO import StringIO except: @@ -62,7 +83,8 @@ else: from urllib import quote_plus, unquote_plus # noqa from htmlentitydefs import codepoint2name, name2codepoint # noqa - string_types = basestring, # noqa + + string_types = (basestring,) # noqa binary_type = str text_type = unicode # noqa @@ -76,16 +98,18 @@ else: return eval("0" + lit) -if py33: +if py3k: from importlib import machinery def load_module(module_id, path): return machinery.SourceFileLoader(module_id, path).load_module() + + else: import imp def load_module(module_id, path): - fp = open(path, 'rb') + fp = open(path, "rb") try: return imp.load_source(module_id, path, fp) finally: @@ -93,93 +117,32 @@ else: if py3k: + def reraise(tp, value, tb=None, cause=None): if cause is not None: value.__cause__ = cause if value.__traceback__ is not tb: raise value.with_traceback(tb) raise value + + else: - exec("def reraise(tp, value, tb=None, cause=None):\n" - " raise tp, value, tb\n") + exec( + "def reraise(tp, value, tb=None, cause=None):\n" + " raise tp, value, tb\n" + ) def exception_as(): return sys.exc_info()[1] -try: - import threading - if py3k: - import _thread as thread - else: - import thread -except ImportError: - import dummy_threading as threading # noqa - if py3k: - import _dummy_thread as thread - else: - import dummy_thread as thread # noqa -if win32 or jython: - time_func = time.clock -else: - time_func = time.time - -try: - from functools import partial -except: - def partial(func, *args, **keywords): - def newfunc(*fargs, **fkeywords): - newkeywords = keywords.copy() - newkeywords.update(fkeywords) - return func(*(args + fargs), **newkeywords) - return newfunc - - -all = all -import json # noqa +all = all # noqa def exception_name(exc): return exc.__class__.__name__ -try: - from inspect import CO_VARKEYWORDS, CO_VARARGS - - def inspect_func_args(fn): - if py3k: - co = fn.__code__ - else: - co = fn.func_code - - nargs = co.co_argcount - names = co.co_varnames - args = list(names[:nargs]) - - varargs = None - if co.co_flags & CO_VARARGS: - varargs = co.co_varnames[nargs] - nargs = nargs + 1 - varkw = None - if co.co_flags & CO_VARKEYWORDS: - varkw = co.co_varnames[nargs] - - if py3k: - return args, varargs, varkw, fn.__defaults__ - else: - return args, varargs, varkw, fn.func_defaults -except ImportError: - import inspect - - def inspect_func_args(fn): - return inspect.getargspec(fn) - -if py3k: - def callable(fn): - return hasattr(fn, '__call__') -else: - callable = callable - ################################################ # cross-compatible metaclass implementation @@ -187,6 +150,8 @@ else: def with_metaclass(meta, base=object): """Create a base class with a metaclass.""" return meta("%sBase" % meta.__name__, (base,), {}) + + ################################################ @@ -195,7 +160,7 @@ def arg_stringname(func_arg): In Python3.4 a function's args are of _ast.arg type not _ast.name """ - if hasattr(func_arg, 'arg'): + if hasattr(func_arg, "arg"): return func_arg.arg else: return str(func_arg) diff --git a/server/www/packages/packages-linux/x64/mako/exceptions.py b/server/www/packages/packages-linux/x64/mako/exceptions.py index cb6fb3f..b6388b1 100644 --- a/server/www/packages/packages-linux/x64/mako/exceptions.py +++ b/server/www/packages/packages-linux/x64/mako/exceptions.py @@ -1,14 +1,16 @@ # mako/exceptions.py -# Copyright (C) 2006-2016 the Mako authors and contributors +# Copyright 2006-2019 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php """exception classes""" -import traceback import sys -from mako import util, compat +import traceback + +from mako import compat +from mako import util class MakoException(Exception): @@ -27,11 +29,10 @@ def _format_filepos(lineno, pos, filename): class CompileException(MakoException): - def __init__(self, message, source, lineno, pos, filename): MakoException.__init__( - self, - message + _format_filepos(lineno, pos, filename)) + self, message + _format_filepos(lineno, pos, filename) + ) self.lineno = lineno self.pos = pos self.filename = filename @@ -39,11 +40,10 @@ class CompileException(MakoException): class SyntaxException(MakoException): - def __init__(self, message, source, lineno, pos, filename): MakoException.__init__( - self, - message + _format_filepos(lineno, pos, filename)) + self, message + _format_filepos(lineno, pos, filename) + ) self.lineno = lineno self.pos = pos self.filename = filename @@ -115,7 +115,7 @@ class RichTraceback(object): # str(Exception(u'\xe6')) work in Python < 2.6 self.message = self.error.args[0] if not isinstance(self.message, compat.text_type): - self.message = compat.text_type(self.message, 'ascii', 'replace') + self.message = compat.text_type(self.message, "ascii", "replace") def _get_reformatted_records(self, records): for rec in records: @@ -151,25 +151,30 @@ class RichTraceback(object): source, and code line from that line number of the template.""" import mako.template + mods = {} rawrecords = traceback.extract_tb(trcback) new_trcback = [] for filename, lineno, function, line in rawrecords: if not line: - line = '' + line = "" try: - (line_map, template_lines) = mods[filename] + (line_map, template_lines, template_filename) = mods[filename] except KeyError: try: info = mako.template._get_module_info(filename) module_source = info.code template_source = info.source - template_filename = info.template_filename or filename + template_filename = ( + info.template_filename + or info.template_uri + or filename + ) except KeyError: # A normal .py file (not a Template) if not compat.py3k: try: - fp = open(filename, 'rb') + fp = open(filename, "rb") encoding = util.parse_encoding(fp) fp.close() except IOError: @@ -177,21 +182,33 @@ class RichTraceback(object): if encoding: line = line.decode(encoding) else: - line = line.decode('ascii', 'replace') - new_trcback.append((filename, lineno, function, line, - None, None, None, None)) + line = line.decode("ascii", "replace") + new_trcback.append( + ( + filename, + lineno, + function, + line, + None, + None, + None, + None, + ) + ) continue template_ln = 1 - source_map = mako.template.ModuleInfo.\ - get_module_source_metadata( - module_source, full_line_map=True) - line_map = source_map['full_line_map'] + mtm = mako.template.ModuleInfo + source_map = mtm.get_module_source_metadata( + module_source, full_line_map=True + ) + line_map = source_map["full_line_map"] - template_lines = [line_ for line_ in - template_source.split("\n")] - mods[filename] = (line_map, template_lines) + template_lines = [ + line_ for line_ in template_source.split("\n") + ] + mods[filename] = (line_map, template_lines, template_filename) template_ln = line_map[lineno - 1] @@ -199,9 +216,18 @@ class RichTraceback(object): template_line = template_lines[template_ln - 1] else: template_line = None - new_trcback.append((filename, lineno, function, - line, template_filename, template_ln, - template_line, template_source)) + new_trcback.append( + ( + filename, + lineno, + function, + line, + template_filename, + template_ln, + template_line, + template_source, + ) + ) if not self.source: for l in range(len(new_trcback) - 1, 0, -1): if new_trcback[l][5]: @@ -212,15 +238,17 @@ class RichTraceback(object): if new_trcback: try: # A normal .py file (not a Template) - fp = open(new_trcback[-1][0], 'rb') + fp = open(new_trcback[-1][0], "rb") encoding = util.parse_encoding(fp) + if compat.py3k and not encoding: + encoding = "utf-8" fp.seek(0) self.source = fp.read() fp.close() if encoding: self.source = self.source.decode(encoding) except IOError: - self.source = '' + self.source = "" self.lineno = new_trcback[-1][1] return new_trcback @@ -233,7 +261,9 @@ def text_error_template(lookup=None): """ import mako.template - return mako.template.Template(r""" + + return mako.template.Template( + r""" <%page args="error=None, traceback=None"/> <%! from mako.exceptions import RichTraceback @@ -247,7 +277,8 @@ Traceback (most recent call last): ${line | trim} % endfor ${tback.errorname}: ${tback.message} -""") +""" + ) def _install_pygments(): @@ -259,9 +290,10 @@ def _install_pygments(): def _install_fallback(): global syntax_highlight, pygments_html_formatter from mako.filters import html_escape + pygments_html_formatter = None - def syntax_highlight(filename='', language=None): + def syntax_highlight(filename="", language=None): return html_escape @@ -270,6 +302,8 @@ def _install_highlighting(): _install_pygments() except ImportError: _install_fallback() + + _install_highlighting() @@ -287,7 +321,9 @@ def html_error_template(): """ import mako.template - return mako.template.Template(r""" + + return mako.template.Template( + r""" <%! from mako.exceptions import RichTraceback, syntax_highlight,\ pygments_html_formatter @@ -390,5 +426,7 @@ def html_error_template(): % endif -""", output_encoding=sys.getdefaultencoding(), - encoding_errors='htmlentityreplace') +""", + output_encoding=sys.getdefaultencoding(), + encoding_errors="htmlentityreplace", + ) diff --git a/server/www/packages/packages-linux/x64/mako/ext/autohandler.py b/server/www/packages/packages-linux/x64/mako/ext/autohandler.py index 9d1c911..55afb95 100644 --- a/server/www/packages/packages-linux/x64/mako/ext/autohandler.py +++ b/server/www/packages/packages-linux/x64/mako/ext/autohandler.py @@ -1,5 +1,5 @@ # ext/autohandler.py -# Copyright (C) 2006-2016 the Mako authors and contributors +# Copyright 2006-2019 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -8,29 +8,29 @@ requires that the TemplateLookup class is used with templates. -usage: +usage:: -<%! - from mako.ext.autohandler import autohandler -%> -<%inherit file="${autohandler(template, context)}"/> + <%! + from mako.ext.autohandler import autohandler + %> + <%inherit file="${autohandler(template, context)}"/> -or with custom autohandler filename: +or with custom autohandler filename:: -<%! - from mako.ext.autohandler import autohandler -%> -<%inherit file="${autohandler(template, context, name='somefilename')}"/> + <%! + from mako.ext.autohandler import autohandler + %> + <%inherit file="${autohandler(template, context, name='somefilename')}"/> """ -import posixpath import os +import posixpath import re -def autohandler(template, context, name='autohandler'): +def autohandler(template, context, name="autohandler"): lookup = context.lookup _template_uri = template.module._template_uri if not lookup.filesystem_checks: @@ -39,13 +39,14 @@ def autohandler(template, context, name='autohandler'): except KeyError: pass - tokens = re.findall(r'([^/]+)', posixpath.dirname(_template_uri)) + [name] + tokens = re.findall(r"([^/]+)", posixpath.dirname(_template_uri)) + [name] while len(tokens): - path = '/' + '/'.join(tokens) + path = "/" + "/".join(tokens) if path != _template_uri and _file_exists(lookup, path): if not lookup.filesystem_checks: return lookup._uri_cache.setdefault( - (autohandler, _template_uri, name), path) + (autohandler, _template_uri, name), path + ) else: return path if len(tokens) == 1: @@ -54,15 +55,16 @@ def autohandler(template, context, name='autohandler'): if not lookup.filesystem_checks: return lookup._uri_cache.setdefault( - (autohandler, _template_uri, name), None) + (autohandler, _template_uri, name), None + ) else: return None def _file_exists(lookup, path): - psub = re.sub(r'^/', '', path) + psub = re.sub(r"^/", "", path) for d in lookup.directories: - if os.path.exists(d + '/' + psub): + if os.path.exists(d + "/" + psub): return True else: return False diff --git a/server/www/packages/packages-linux/x64/mako/ext/babelplugin.py b/server/www/packages/packages-linux/x64/mako/ext/babelplugin.py index 0b5e84f..dbe2cd0 100644 --- a/server/www/packages/packages-linux/x64/mako/ext/babelplugin.py +++ b/server/www/packages/packages-linux/x64/mako/ext/babelplugin.py @@ -1,23 +1,24 @@ # ext/babelplugin.py -# Copyright (C) 2006-2016 the Mako authors and contributors +# Copyright 2006-2019 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php """gettext message extraction via Babel: http://babel.edgewall.org/""" from babel.messages.extract import extract_python + from mako.ext.extract import MessageExtractor class BabelMakoExtractor(MessageExtractor): - def __init__(self, keywords, comment_tags, options): self.keywords = keywords self.options = options self.config = { - 'comment-tags': u' '.join(comment_tags), - 'encoding': options.get('input_encoding', - options.get('encoding', None)), + "comment-tags": u" ".join(comment_tags), + "encoding": options.get( + "input_encoding", options.get("encoding", None) + ), } super(BabelMakoExtractor, self).__init__() @@ -25,12 +26,19 @@ class BabelMakoExtractor(MessageExtractor): return self.process_file(fileobj) def process_python(self, code, code_lineno, translator_strings): - comment_tags = self.config['comment-tags'] - for lineno, funcname, messages, python_translator_comments \ - in extract_python(code, - self.keywords, comment_tags, self.options): - yield (code_lineno + (lineno - 1), funcname, messages, - translator_strings + python_translator_comments) + comment_tags = self.config["comment-tags"] + for ( + lineno, + funcname, + messages, + python_translator_comments, + ) in extract_python(code, self.keywords, comment_tags, self.options): + yield ( + code_lineno + (lineno - 1), + funcname, + messages, + translator_strings + python_translator_comments, + ) def extract(fileobj, keywords, comment_tags, options): diff --git a/server/www/packages/packages-linux/x64/mako/ext/beaker_cache.py b/server/www/packages/packages-linux/x64/mako/ext/beaker_cache.py index c7c260d..b415c9c 100644 --- a/server/www/packages/packages-linux/x64/mako/ext/beaker_cache.py +++ b/server/www/packages/packages-linux/x64/mako/ext/beaker_cache.py @@ -1,7 +1,12 @@ +# ext/beaker_cache.py +# Copyright 2006-2019 the Mako authors and contributors +# +# This module is part of Mako and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + """Provide a :class:`.CacheImpl` for the Beaker caching system.""" from mako import exceptions - from mako.cache import CacheImpl try: @@ -27,36 +32,37 @@ class BeakerCacheImpl(CacheImpl): def __init__(self, cache): if not has_beaker: raise exceptions.RuntimeException( - "Can't initialize Beaker plugin; Beaker is not installed.") + "Can't initialize Beaker plugin; Beaker is not installed." + ) global _beaker_cache if _beaker_cache is None: - if 'manager' in cache.template.cache_args: - _beaker_cache = cache.template.cache_args['manager'] + if "manager" in cache.template.cache_args: + _beaker_cache = cache.template.cache_args["manager"] else: _beaker_cache = beaker_cache.CacheManager() super(BeakerCacheImpl, self).__init__(cache) def _get_cache(self, **kw): - expiretime = kw.pop('timeout', None) - if 'dir' in kw: - kw['data_dir'] = kw.pop('dir') + expiretime = kw.pop("timeout", None) + if "dir" in kw: + kw["data_dir"] = kw.pop("dir") elif self.cache.template.module_directory: - kw['data_dir'] = self.cache.template.module_directory + kw["data_dir"] = self.cache.template.module_directory - if 'manager' in kw: - kw.pop('manager') + if "manager" in kw: + kw.pop("manager") - if kw.get('type') == 'memcached': - kw['type'] = 'ext:memcached' + if kw.get("type") == "memcached": + kw["type"] = "ext:memcached" - if 'region' in kw: - region = kw.pop('region') + if "region" in kw: + region = kw.pop("region") cache = _beaker_cache.get_cache_region(self.cache.id, region, **kw) else: cache = _beaker_cache.get_cache(self.cache.id, **kw) - cache_args = {'starttime': self.cache.starttime} + cache_args = {"starttime": self.cache.starttime} if expiretime: - cache_args['expiretime'] = expiretime + cache_args["expiretime"] = expiretime return cache, cache_args def get_or_create(self, key, creation_function, **kw): diff --git a/server/www/packages/packages-linux/x64/mako/ext/extract.py b/server/www/packages/packages-linux/x64/mako/ext/extract.py index d777ea8..8a1bd54 100644 --- a/server/www/packages/packages-linux/x64/mako/ext/extract.py +++ b/server/www/packages/packages-linux/x64/mako/ext/extract.py @@ -1,30 +1,39 @@ +# ext/extract.py +# Copyright 2006-2019 the Mako authors and contributors +# +# This module is part of Mako and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + import re + from mako import compat from mako import lexer from mako import parsetree class MessageExtractor(object): - def process_file(self, fileobj): template_node = lexer.Lexer( - fileobj.read(), - input_encoding=self.config['encoding']).parse() + fileobj.read(), input_encoding=self.config["encoding"] + ).parse() for extracted in self.extract_nodes(template_node.get_children()): yield extracted def extract_nodes(self, nodes): translator_comments = [] in_translator_comments = False - input_encoding = self.config['encoding'] or 'ascii' + input_encoding = self.config["encoding"] or "ascii" comment_tags = list( - filter(None, re.split(r'\s+', self.config['comment-tags']))) + filter(None, re.split(r"\s+", self.config["comment-tags"])) + ) for node in nodes: child_nodes = None - if in_translator_comments and \ - isinstance(node, parsetree.Text) and \ - not node.content.strip(): + if ( + in_translator_comments + and isinstance(node, parsetree.Text) + and not node.content.strip() + ): # Ignore whitespace within translator comments continue @@ -32,13 +41,15 @@ class MessageExtractor(object): value = node.text.strip() if in_translator_comments: translator_comments.extend( - self._split_comment(node.lineno, value)) + self._split_comment(node.lineno, value) + ) continue for comment_tag in comment_tags: if value.startswith(comment_tag): in_translator_comments = True translator_comments.extend( - self._split_comment(node.lineno, value)) + self._split_comment(node.lineno, value) + ) continue if isinstance(node, parsetree.DefTag): @@ -69,15 +80,18 @@ class MessageExtractor(object): continue # Comments don't apply unless they immediately precede the message - if translator_comments and \ - translator_comments[-1][0] < node.lineno - 1: + if ( + translator_comments + and translator_comments[-1][0] < node.lineno - 1 + ): translator_comments = [] translator_strings = [ - comment[1] for comment in translator_comments] + comment[1] for comment in translator_comments + ] if isinstance(code, compat.text_type): - code = code.encode(input_encoding, 'backslashreplace') + code = code.encode(input_encoding, "backslashreplace") used_translator_comments = False # We add extra newline to work around a pybabel bug @@ -85,10 +99,11 @@ class MessageExtractor(object): # input string of the input is non-ascii) # Also, because we added it, we have to subtract one from # node.lineno - code = compat.byte_buffer(compat.b('\n') + code) + code = compat.byte_buffer(compat.b("\n") + code) for message in self.process_python( - code, node.lineno - 1, translator_strings): + code, node.lineno - 1, translator_strings + ): yield message used_translator_comments = True @@ -104,5 +119,7 @@ class MessageExtractor(object): def _split_comment(lineno, comment): """Return the multiline comment at lineno split into a list of comment line numbers and the accompanying comment line""" - return [(lineno + index, line) for index, line in - enumerate(comment.splitlines())] + return [ + (lineno + index, line) + for index, line in enumerate(comment.splitlines()) + ] diff --git a/server/www/packages/packages-linux/x64/mako/ext/linguaplugin.py b/server/www/packages/packages-linux/x64/mako/ext/linguaplugin.py index 46b0d6a..955a5cb 100644 --- a/server/www/packages/packages-linux/x64/mako/ext/linguaplugin.py +++ b/server/www/packages/packages-linux/x64/mako/ext/linguaplugin.py @@ -1,43 +1,57 @@ +# ext/linguaplugin.py +# Copyright 2006-2019 the Mako authors and contributors +# +# This module is part of Mako and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + import io + from lingua.extractors import Extractor -from lingua.extractors import Message from lingua.extractors import get_extractor -from mako.ext.extract import MessageExtractor +from lingua.extractors import Message + from mako import compat +from mako.ext.extract import MessageExtractor class LinguaMakoExtractor(Extractor, MessageExtractor): - '''Mako templates''' - extensions = ['.mako'] - default_config = { - 'encoding': 'utf-8', - 'comment-tags': '', - } + """Mako templates""" + + extensions = [".mako"] + default_config = {"encoding": "utf-8", "comment-tags": ""} def __call__(self, filename, options, fileobj=None): self.options = options self.filename = filename - self.python_extractor = get_extractor('x.py') + self.python_extractor = get_extractor("x.py") if fileobj is None: - fileobj = open(filename, 'rb') + fileobj = open(filename, "rb") return self.process_file(fileobj) def process_python(self, code, code_lineno, translator_strings): source = code.getvalue().strip() - if source.endswith(compat.b(':')): - if source in (compat.b('try:'), compat.b('else:')) or source.startswith(compat.b('except')): - source = compat.b('') # Ignore try/except and else - elif source.startswith(compat.b('elif')): - source = source[2:] # Replace "elif" with "if" - source += compat.b('pass') + if source.endswith(compat.b(":")): + if source in ( + compat.b("try:"), + compat.b("else:"), + ) or source.startswith(compat.b("except")): + source = compat.b("") # Ignore try/except and else + elif source.startswith(compat.b("elif")): + source = source[2:] # Replace "elif" with "if" + source += compat.b("pass") code = io.BytesIO(source) for msg in self.python_extractor( - self.filename, self.options, code, code_lineno -1): + self.filename, self.options, code, code_lineno - 1 + ): if translator_strings: - msg = Message(msg.msgctxt, msg.msgid, msg.msgid_plural, - msg.flags, - compat.u(' ').join( - translator_strings + [msg.comment]), - msg.tcomment, msg.location) + msg = Message( + msg.msgctxt, + msg.msgid, + msg.msgid_plural, + msg.flags, + compat.u(" ").join(translator_strings + [msg.comment]), + msg.tcomment, + msg.location, + ) yield msg diff --git a/server/www/packages/packages-linux/x64/mako/ext/preprocessors.py b/server/www/packages/packages-linux/x64/mako/ext/preprocessors.py index 9b700d1..1eeb7c5 100644 --- a/server/www/packages/packages-linux/x64/mako/ext/preprocessors.py +++ b/server/www/packages/packages-linux/x64/mako/ext/preprocessors.py @@ -1,5 +1,5 @@ # ext/preprocessors.py -# Copyright (C) 2006-2016 the Mako authors and contributors +# Copyright 2006-2019 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -17,4 +17,4 @@ def convert_comments(text): from mako.ext.preprocessors import convert_comments t = Template(..., preprocessor=convert_comments)""" - return re.sub(r'(?<=\n)\s*#[^#]', "##", text) + return re.sub(r"(?<=\n)\s*#[^#]", "##", text) diff --git a/server/www/packages/packages-linux/x64/mako/ext/pygmentplugin.py b/server/www/packages/packages-linux/x64/mako/ext/pygmentplugin.py index 4057caa..1734ccd 100644 --- a/server/www/packages/packages-linux/x64/mako/ext/pygmentplugin.py +++ b/server/www/packages/packages-linux/x64/mako/ext/pygmentplugin.py @@ -1,45 +1,73 @@ # ext/pygmentplugin.py -# Copyright (C) 2006-2016 the Mako authors and contributors +# Copyright 2006-2019 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -from pygments.lexers.web import \ - HtmlLexer, XmlLexer, JavascriptLexer, CssLexer -from pygments.lexers.agile import PythonLexer, Python3Lexer -from pygments.lexer import DelegatingLexer, RegexLexer, bygroups, \ - include, using -from pygments.token import \ - Text, Comment, Operator, Keyword, Name, String, Other -from pygments.formatters.html import HtmlFormatter from pygments import highlight +from pygments.formatters.html import HtmlFormatter +from pygments.lexer import bygroups +from pygments.lexer import DelegatingLexer +from pygments.lexer import include +from pygments.lexer import RegexLexer +from pygments.lexer import using +from pygments.lexers.agile import Python3Lexer +from pygments.lexers.agile import PythonLexer +from pygments.lexers.web import CssLexer +from pygments.lexers.web import HtmlLexer +from pygments.lexers.web import JavascriptLexer +from pygments.lexers.web import XmlLexer +from pygments.token import Comment +from pygments.token import Keyword +from pygments.token import Name +from pygments.token import Operator +from pygments.token import Other +from pygments.token import String +from pygments.token import Text + from mako import compat class MakoLexer(RegexLexer): - name = 'Mako' - aliases = ['mako'] - filenames = ['*.mao'] + name = "Mako" + aliases = ["mako"] + filenames = ["*.mao"] tokens = { - 'root': [ - (r'(\s*)(\%)(\s*end(?:\w+))(\n|\Z)', - bygroups(Text, Comment.Preproc, Keyword, Other)), - (r'(\s*)(\%(?!%))([^\n]*)(\n|\Z)', - bygroups(Text, Comment.Preproc, using(PythonLexer), Other)), - (r'(\s*)(##[^\n]*)(\n|\Z)', - bygroups(Text, Comment.Preproc, Other)), - (r'''(?s)<%doc>.*?''', Comment.Preproc), - (r'(<%)([\w\.\:]+)', - bygroups(Comment.Preproc, Name.Builtin), 'tag'), - (r'()', - bygroups(Comment.Preproc, Name.Builtin, Comment.Preproc)), - (r'<%(?=([\w\.\:]+))', Comment.Preproc, 'ondeftags'), - (r'(<%(?:!?))(.*?)(%>)(?s)', - bygroups(Comment.Preproc, using(PythonLexer), Comment.Preproc)), - (r'(\$\{)(.*?)(\})', - bygroups(Comment.Preproc, using(PythonLexer), Comment.Preproc)), - (r'''(?sx) + "root": [ + ( + r"(\s*)(\%)(\s*end(?:\w+))(\n|\Z)", + bygroups(Text, Comment.Preproc, Keyword, Other), + ), + ( + r"(\s*)(\%(?!%))([^\n]*)(\n|\Z)", + bygroups(Text, Comment.Preproc, using(PythonLexer), Other), + ), + ( + r"(\s*)(##[^\n]*)(\n|\Z)", + bygroups(Text, Comment.Preproc, Other), + ), + (r"""(?s)<%doc>.*?""", Comment.Preproc), + ( + r"(<%)([\w\.\:]+)", + bygroups(Comment.Preproc, Name.Builtin), + "tag", + ), + ( + r"()", + bygroups(Comment.Preproc, Name.Builtin, Comment.Preproc), + ), + (r"<%(?=([\w\.\:]+))", Comment.Preproc, "ondeftags"), + ( + r"(?s)(<%(?:!?))(.*?)(%>)", + bygroups(Comment.Preproc, using(PythonLexer), Comment.Preproc), + ), + ( + r"(\$\{)(.*?)(\})", + bygroups(Comment.Preproc, using(PythonLexer), Comment.Preproc), + ), + ( + r"""(?sx) (.+?) # anything, followed by: (?: (?<=\n)(?=%(?!%)|\#\#) | # an eval or comment line @@ -52,76 +80,78 @@ class MakoLexer(RegexLexer): (\\\n) | # an escaped newline \Z # end of string ) - ''', bygroups(Other, Operator)), - (r'\s+', Text), + """, + bygroups(Other, Operator), + ), + (r"\s+", Text), ], - 'ondeftags': [ - (r'<%', Comment.Preproc), - (r'(?<=<%)(include|inherit|namespace|page)', Name.Builtin), - include('tag'), + "ondeftags": [ + (r"<%", Comment.Preproc), + (r"(?<=<%)(include|inherit|namespace|page)", Name.Builtin), + include("tag"), ], - 'tag': [ - (r'((?:\w+)\s*=)\s*(".*?")', - bygroups(Name.Attribute, String)), - (r'/?\s*>', Comment.Preproc, '#pop'), - (r'\s+', Text), + "tag": [ + (r'((?:\w+)\s*=)\s*(".*?")', bygroups(Name.Attribute, String)), + (r"/?\s*>", Comment.Preproc, "#pop"), + (r"\s+", Text), ], - 'attr': [ - ('".*?"', String, '#pop'), - ("'.*?'", String, '#pop'), - (r'[^\s>]+', String, '#pop'), + "attr": [ + ('".*?"', String, "#pop"), + ("'.*?'", String, "#pop"), + (r"[^\s>]+", String, "#pop"), ], } class MakoHtmlLexer(DelegatingLexer): - name = 'HTML+Mako' - aliases = ['html+mako'] + name = "HTML+Mako" + aliases = ["html+mako"] def __init__(self, **options): - super(MakoHtmlLexer, self).__init__(HtmlLexer, MakoLexer, - **options) + super(MakoHtmlLexer, self).__init__(HtmlLexer, MakoLexer, **options) class MakoXmlLexer(DelegatingLexer): - name = 'XML+Mako' - aliases = ['xml+mako'] + name = "XML+Mako" + aliases = ["xml+mako"] def __init__(self, **options): - super(MakoXmlLexer, self).__init__(XmlLexer, MakoLexer, - **options) + super(MakoXmlLexer, self).__init__(XmlLexer, MakoLexer, **options) class MakoJavascriptLexer(DelegatingLexer): - name = 'JavaScript+Mako' - aliases = ['js+mako', 'javascript+mako'] + name = "JavaScript+Mako" + aliases = ["js+mako", "javascript+mako"] def __init__(self, **options): - super(MakoJavascriptLexer, self).__init__(JavascriptLexer, - MakoLexer, **options) + super(MakoJavascriptLexer, self).__init__( + JavascriptLexer, MakoLexer, **options + ) class MakoCssLexer(DelegatingLexer): - name = 'CSS+Mako' - aliases = ['css+mako'] + name = "CSS+Mako" + aliases = ["css+mako"] def __init__(self, **options): - super(MakoCssLexer, self).__init__(CssLexer, MakoLexer, - **options) + super(MakoCssLexer, self).__init__(CssLexer, MakoLexer, **options) -pygments_html_formatter = HtmlFormatter(cssclass='syntax-highlighted', - linenos=True) +pygments_html_formatter = HtmlFormatter( + cssclass="syntax-highlighted", linenos=True +) -def syntax_highlight(filename='', language=None): +def syntax_highlight(filename="", language=None): mako_lexer = MakoLexer() if compat.py3k: python_lexer = Python3Lexer() else: python_lexer = PythonLexer() - if filename.startswith('memory:') or language == 'mako': - return lambda string: highlight(string, mako_lexer, - pygments_html_formatter) - return lambda string: highlight(string, python_lexer, - pygments_html_formatter) + if filename.startswith("memory:") or language == "mako": + return lambda string: highlight( + string, mako_lexer, pygments_html_formatter + ) + return lambda string: highlight( + string, python_lexer, pygments_html_formatter + ) diff --git a/server/www/packages/packages-linux/x64/mako/ext/turbogears.py b/server/www/packages/packages-linux/x64/mako/ext/turbogears.py index eaa2d78..fdb7741 100644 --- a/server/www/packages/packages-linux/x64/mako/ext/turbogears.py +++ b/server/www/packages/packages-linux/x64/mako/ext/turbogears.py @@ -1,5 +1,5 @@ # ext/turbogears.py -# Copyright (C) 2006-2016 the Mako authors and contributors +# Copyright 2006-2019 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -13,7 +13,7 @@ class TGPlugin(object): """TurboGears compatible Template Plugin.""" - def __init__(self, extra_vars_func=None, options=None, extension='mak'): + def __init__(self, extra_vars_func=None, options=None, extension="mak"): self.extra_vars_func = extra_vars_func self.extension = extension if not options: @@ -22,9 +22,9 @@ class TGPlugin(object): # Pull the options out and initialize the lookup lookup_options = {} for k, v in options.items(): - if k.startswith('mako.'): + if k.startswith("mako."): lookup_options[k[5:]] = v - elif k in ['directories', 'filesystem_checks', 'module_directory']: + elif k in ["directories", "filesystem_checks", "module_directory"]: lookup_options[k] = v self.lookup = TemplateLookup(**lookup_options) @@ -40,14 +40,17 @@ class TGPlugin(object): if template_string is not None: return Template(template_string, **self.tmpl_options) # Translate TG dot notation to normal / template path - if '/' not in templatename: - templatename = '/' + templatename.replace('.', '/') + '.' +\ - self.extension + if "/" not in templatename: + templatename = ( + "/" + templatename.replace(".", "/") + "." + self.extension + ) # Lookup template return self.lookup.get_template(templatename) - def render(self, info, format="html", fragment=False, template=None): + def render( + self, info, format="html", fragment=False, template=None # noqa + ): if isinstance(template, compat.string_types): template = self.load_template(template) diff --git a/server/www/packages/packages-linux/x64/mako/filters.py b/server/www/packages/packages-linux/x64/mako/filters.py index c082690..ba69fdd 100644 --- a/server/www/packages/packages-linux/x64/mako/filters.py +++ b/server/www/packages/packages-linux/x64/mako/filters.py @@ -1,24 +1,25 @@ # mako/filters.py -# Copyright (C) 2006-2016 the Mako authors and contributors +# Copyright 2006-2019 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -import re import codecs - -from mako.compat import quote_plus, unquote_plus, codepoint2name, \ - name2codepoint +import re from mako import compat +from mako.compat import codepoint2name +from mako.compat import name2codepoint +from mako.compat import quote_plus +from mako.compat import unquote_plus xml_escapes = { - '&': '&', - '>': '>', - '<': '<', - '"': '"', # also " in html-only - "'": ''' # also ' in html-only + "&": "&", + ">": ">", + "<": "<", + '"': """, # also " in html-only + "'": "'", # also ' in html-only } # XXX: " is valid in HTML and XML @@ -37,6 +38,7 @@ def legacy_html_escape(s): try: import markupsafe + html_escape = markupsafe.escape except ImportError: html_escape = legacy_html_escape @@ -69,7 +71,6 @@ def trim(string): class Decode(object): - def __getattr__(self, key): def decode(x): if isinstance(x, compat.text_type): @@ -78,24 +79,31 @@ class Decode(object): return decode(str(x)) else: return compat.text_type(x, encoding=key) + return decode + + decode = Decode() -_ASCII_re = re.compile(r'\A[\x00-\x7f]*\Z') +_ASCII_re = re.compile(r"\A[\x00-\x7f]*\Z") def is_ascii_str(text): return isinstance(text, str) and _ASCII_re.match(text) + ################################################################ class XMLEntityEscaper(object): - def __init__(self, codepoint2name, name2codepoint): - self.codepoint2entity = dict([(c, compat.text_type('&%s;' % n)) - for c, n in codepoint2name.items()]) + self.codepoint2entity = dict( + [ + (c, compat.text_type("&%s;" % n)) + for c, n in codepoint2name.items() + ] + ) self.name2codepoint = name2codepoint def escape_entities(self, text): @@ -110,7 +118,7 @@ class XMLEntityEscaper(object): try: return self.codepoint2entity[codepoint] except (KeyError, IndexError): - return '&#x%X;' % codepoint + return "&#x%X;" % codepoint __escapable = re.compile(r'["&<>]|[^\x00-\x7f]') @@ -123,19 +131,22 @@ class XMLEntityEscaper(object): The return value is guaranteed to be ASCII. """ - return self.__escapable.sub(self.__escape, compat.text_type(text) - ).encode('ascii') + return self.__escapable.sub( + self.__escape, compat.text_type(text) + ).encode("ascii") # XXX: This regexp will not match all valid XML entity names__. # (It punts on details involving involving CombiningChars and Extenders.) # # .. __: http://www.w3.org/TR/2000/REC-xml-20001006#NT-EntityRef - __characterrefs = re.compile(r'''& (?: + __characterrefs = re.compile( + r"""& (?: \#(\d+) | \#x([\da-f]+) | ( (?!\d) [:\w] [-.:\w]+ ) - ) ;''', - re.X | re.UNICODE) + ) ;""", + re.X | re.UNICODE, + ) def __unescape(self, m): dval, hval, name = m.groups() @@ -144,7 +155,7 @@ class XMLEntityEscaper(object): elif hval: codepoint = int(hval, 16) else: - codepoint = self.name2codepoint.get(name, 0xfffd) + codepoint = self.name2codepoint.get(name, 0xFFFD) # U+FFFD = "REPLACEMENT CHARACTER" if codepoint < 128: return chr(codepoint) @@ -168,42 +179,41 @@ html_entities_unescape = _html_entities_escaper.unescape def htmlentityreplace_errors(ex): """An encoding error handler. - This python `codecs`_ error handler replaces unencodable + This python codecs error handler replaces unencodable characters with HTML entities, or, if no HTML entity exists for - the character, XML character references. + the character, XML character references:: - >>> u'The cost was \u20ac12.'.encode('latin1', 'htmlentityreplace') - 'The cost was €12.' + >>> u'The cost was \u20ac12.'.encode('latin1', 'htmlentityreplace') + 'The cost was €12.' """ if isinstance(ex, UnicodeEncodeError): # Handle encoding errors - bad_text = ex.object[ex.start:ex.end] + bad_text = ex.object[ex.start : ex.end] text = _html_entities_escaper.escape(bad_text) return (compat.text_type(text), ex.end) raise ex -codecs.register_error('htmlentityreplace', htmlentityreplace_errors) + +codecs.register_error("htmlentityreplace", htmlentityreplace_errors) # TODO: options to make this dynamic per-compilation will be added in a later # release DEFAULT_ESCAPES = { - 'x': 'filters.xml_escape', - 'h': 'filters.html_escape', - 'u': 'filters.url_escape', - 'trim': 'filters.trim', - 'entity': 'filters.html_entities_escape', - 'unicode': 'unicode', - 'decode': 'decode', - 'str': 'str', - 'n': 'n' + "x": "filters.xml_escape", + "h": "filters.html_escape", + "u": "filters.url_escape", + "trim": "filters.trim", + "entity": "filters.html_entities_escape", + "unicode": "unicode", + "decode": "decode", + "str": "str", + "n": "n", } if compat.py3k: - DEFAULT_ESCAPES.update({ - 'unicode': 'str' - }) + DEFAULT_ESCAPES.update({"unicode": "str"}) NON_UNICODE_ESCAPES = DEFAULT_ESCAPES.copy() -NON_UNICODE_ESCAPES['h'] = 'filters.legacy_html_escape' -NON_UNICODE_ESCAPES['u'] = 'filters.legacy_url_escape' +NON_UNICODE_ESCAPES["h"] = "filters.legacy_html_escape" +NON_UNICODE_ESCAPES["u"] = "filters.legacy_url_escape" diff --git a/server/www/packages/packages-linux/x64/mako/lexer.py b/server/www/packages/packages-linux/x64/mako/lexer.py index cf4187f..dadd663 100644 --- a/server/www/packages/packages-linux/x64/mako/lexer.py +++ b/server/www/packages/packages-linux/x64/mako/lexer.py @@ -1,24 +1,31 @@ # mako/lexer.py -# Copyright (C) 2006-2016 the Mako authors and contributors +# Copyright 2006-2019 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php """provides the Lexer class for parsing template strings into parse trees.""" -import re import codecs -from mako import parsetree, exceptions, compat +import re + +from mako import compat +from mako import exceptions +from mako import parsetree from mako.pygen import adjust_whitespace _regexp_cache = {} class Lexer(object): - - def __init__(self, text, filename=None, - disable_unicode=False, - input_encoding=None, preprocessor=None): + def __init__( + self, + text, + filename=None, + disable_unicode=False, + input_encoding=None, + preprocessor=None, + ): self.text = text self.filename = filename self.template = parsetree.TemplateNode(self.filename) @@ -34,22 +41,24 @@ class Lexer(object): if compat.py3k and disable_unicode: raise exceptions.UnsupportedError( - "Mako for Python 3 does not " - "support disabling Unicode") + "Mako for Python 3 does not " "support disabling Unicode" + ) if preprocessor is None: self.preprocessor = [] - elif not hasattr(preprocessor, '__iter__'): + elif not hasattr(preprocessor, "__iter__"): self.preprocessor = [preprocessor] else: self.preprocessor = preprocessor @property def exception_kwargs(self): - return {'source': self.text, - 'lineno': self.matched_lineno, - 'pos': self.matched_charpos, - 'filename': self.filename} + return { + "source": self.text, + "lineno": self.matched_lineno, + "pos": self.matched_charpos, + "filename": self.filename, + } def match(self, regexp, flags=None): """compile the given regexp, cache the reg, and call match_reg().""" @@ -83,9 +92,9 @@ class Lexer(object): else: self.match_position = end self.matched_lineno = self.lineno - lines = re.findall(r"\n", self.text[mp:self.match_position]) + lines = re.findall(r"\n", self.text[mp : self.match_position]) cp = mp - 1 - while (cp >= 0 and cp < self.textlength and self.text[cp] != '\n'): + while cp >= 0 and cp < self.textlength and self.text[cp] != "\n": cp -= 1 self.matched_charpos = mp - cp self.lineno += len(lines) @@ -97,46 +106,49 @@ class Lexer(object): def parse_until_text(self, watch_nesting, *text): startpos = self.match_position - text_re = r'|'.join(text) + text_re = r"|".join(text) brace_level = 0 paren_level = 0 bracket_level = 0 while True: - match = self.match(r'#.*\n') + match = self.match(r"#.*\n") if match: continue - match = self.match(r'(\"\"\"|\'\'\'|\"|\')[^\\]*?(\\.[^\\]*?)*\1', - re.S) + match = self.match( + r"(\"\"\"|\'\'\'|\"|\')[^\\]*?(\\.[^\\]*?)*\1", re.S + ) if match: continue - match = self.match(r'(%s)' % text_re) - if match and not (watch_nesting - and (brace_level > 0 or paren_level > 0 - or bracket_level > 0)): - return \ - self.text[startpos: - self.match_position - len(match.group(1))],\ - match.group(1) + match = self.match(r"(%s)" % text_re) + if match and not ( + watch_nesting + and (brace_level > 0 or paren_level > 0 or bracket_level > 0) + ): + return ( + self.text[ + startpos : self.match_position - len(match.group(1)) + ], + match.group(1), + ) elif not match: match = self.match(r"(.*?)(?=\"|\'|#|%s)" % text_re, re.S) if match: - brace_level += match.group(1).count('{') - brace_level -= match.group(1).count('}') - paren_level += match.group(1).count('(') - paren_level -= match.group(1).count(')') - bracket_level += match.group(1).count('[') - bracket_level -= match.group(1).count(']') + brace_level += match.group(1).count("{") + brace_level -= match.group(1).count("}") + paren_level += match.group(1).count("(") + paren_level -= match.group(1).count(")") + bracket_level += match.group(1).count("[") + bracket_level -= match.group(1).count("]") continue raise exceptions.SyntaxException( - "Expected: %s" % - ','.join(text), - **self.exception_kwargs) + "Expected: %s" % ",".join(text), **self.exception_kwargs + ) def append_node(self, nodecls, *args, **kwargs): - kwargs.setdefault('source', self.text) - kwargs.setdefault('lineno', self.matched_lineno) - kwargs.setdefault('pos', self.matched_charpos) - kwargs['filename'] = self.filename + kwargs.setdefault("source", self.text) + kwargs.setdefault("lineno", self.matched_lineno) + kwargs.setdefault("pos", self.matched_charpos) + kwargs["filename"] = self.filename node = nodecls(*args, **kwargs) if len(self.tag): self.tag[-1].nodes.append(node) @@ -149,8 +161,10 @@ class Lexer(object): if self.control_line: control_frame = self.control_line[-1] control_frame.nodes.append(node) - if not (isinstance(node, parsetree.ControlLine) and - control_frame.is_ternary(node.keyword)): + if not ( + isinstance(node, parsetree.ControlLine) + and control_frame.is_ternary(node.keyword) + ): if self.ternary_stack and self.ternary_stack[-1]: self.ternary_stack[-1][-1].nodes.append(node) if isinstance(node, parsetree.Tag): @@ -164,17 +178,20 @@ class Lexer(object): elif node.is_primary: self.control_line.append(node) self.ternary_stack.append([]) - elif self.control_line and \ - self.control_line[-1].is_ternary(node.keyword): + elif self.control_line and self.control_line[-1].is_ternary( + node.keyword + ): self.ternary_stack[-1].append(node) - elif self.control_line and \ - not self.control_line[-1].is_ternary(node.keyword): + elif self.control_line and not self.control_line[-1].is_ternary( + node.keyword + ): raise exceptions.SyntaxException( - "Keyword '%s' not a legal ternary for keyword '%s'" % - (node.keyword, self.control_line[-1].keyword), - **self.exception_kwargs) + "Keyword '%s' not a legal ternary for keyword '%s'" + % (node.keyword, self.control_line[-1].keyword), + **self.exception_kwargs + ) - _coding_re = re.compile(r'#.*coding[:=]\s*([-\w.]+).*\r?\n') + _coding_re = re.compile(r"#.*coding[:=]\s*([-\w.]+).*\r?\n") def decode_raw_stream(self, text, decode_raw, known_encoding, filename): """given string/unicode or bytes/string, determine encoding @@ -184,44 +201,48 @@ class Lexer(object): """ if isinstance(text, compat.text_type): m = self._coding_re.match(text) - encoding = m and m.group(1) or known_encoding or 'ascii' + encoding = m and m.group(1) or known_encoding or "ascii" return encoding, text if text.startswith(codecs.BOM_UTF8): - text = text[len(codecs.BOM_UTF8):] - parsed_encoding = 'utf-8' - m = self._coding_re.match(text.decode('utf-8', 'ignore')) - if m is not None and m.group(1) != 'utf-8': + text = text[len(codecs.BOM_UTF8) :] + parsed_encoding = "utf-8" + m = self._coding_re.match(text.decode("utf-8", "ignore")) + if m is not None and m.group(1) != "utf-8": raise exceptions.CompileException( "Found utf-8 BOM in file, with conflicting " "magic encoding comment of '%s'" % m.group(1), - text.decode('utf-8', 'ignore'), - 0, 0, filename) + text.decode("utf-8", "ignore"), + 0, + 0, + filename, + ) else: - m = self._coding_re.match(text.decode('utf-8', 'ignore')) + m = self._coding_re.match(text.decode("utf-8", "ignore")) if m: parsed_encoding = m.group(1) else: - parsed_encoding = known_encoding or 'ascii' + parsed_encoding = known_encoding or "ascii" if decode_raw: try: text = text.decode(parsed_encoding) except UnicodeDecodeError: raise exceptions.CompileException( - "Unicode decode operation of encoding '%s' failed" % - parsed_encoding, - text.decode('utf-8', 'ignore'), - 0, 0, filename) + "Unicode decode operation of encoding '%s' failed" + % parsed_encoding, + text.decode("utf-8", "ignore"), + 0, + 0, + filename, + ) return parsed_encoding, text def parse(self): self.encoding, self.text = self.decode_raw_stream( - self.text, - not self.disable_unicode, - self.encoding, - self.filename) + self.text, not self.disable_unicode, self.encoding, self.filename + ) for preproc in self.preprocessor: self.text = preproc(self.text) @@ -232,7 +253,7 @@ class Lexer(object): self.textlength = len(self.text) - while (True): + while True: if self.match_position > self.textlength: break @@ -258,20 +279,24 @@ class Lexer(object): raise exceptions.CompileException("assertion failed") if len(self.tag): - raise exceptions.SyntaxException("Unclosed tag: <%%%s>" % - self.tag[-1].keyword, - **self.exception_kwargs) + raise exceptions.SyntaxException( + "Unclosed tag: <%%%s>" % self.tag[-1].keyword, + **self.exception_kwargs + ) if len(self.control_line): raise exceptions.SyntaxException( - "Unterminated control keyword: '%s'" % - self.control_line[-1].keyword, + "Unterminated control keyword: '%s'" + % self.control_line[-1].keyword, self.text, self.control_line[-1].lineno, - self.control_line[-1].pos, self.filename) + self.control_line[-1].pos, + self.filename, + ) return self.template def match_tag_start(self): - match = self.match(r''' + match = self.match( + r""" \<% # opening tag ([\w\.\:]+) # keyword @@ -283,9 +308,9 @@ class Lexer(object): (/)?> # closing - ''', - - re.I | re.S | re.X) + """, + re.I | re.S | re.X, + ) if match: keyword, attr, isend = match.groups() @@ -293,22 +318,23 @@ class Lexer(object): attributes = {} if attr: for att in re.findall( - r"\s*(\w+)\s*=\s*(?:'([^']*)'|\"([^\"]*)\")", attr): + r"\s*(\w+)\s*=\s*(?:'([^']*)'|\"([^\"]*)\")", attr + ): key, val1, val2 = att text = val1 or val2 - text = text.replace('\r\n', '\n') + text = text.replace("\r\n", "\n") attributes[key] = text self.append_node(parsetree.Tag, keyword, attributes) if isend: self.tag.pop() else: - if keyword == 'text': - match = self.match(r'(.*?)(?=\)', re.S) + if keyword == "text": + match = self.match(r"(.*?)(?=\)", re.S) if not match: raise exceptions.SyntaxException( - "Unclosed tag: <%%%s>" % - self.tag[-1].keyword, - **self.exception_kwargs) + "Unclosed tag: <%%%s>" % self.tag[-1].keyword, + **self.exception_kwargs + ) self.append_node(parsetree.Text, match.group(1)) return self.match_tag_end() return True @@ -316,25 +342,27 @@ class Lexer(object): return False def match_tag_end(self): - match = self.match(r'\') + match = self.match(r"\") if match: if not len(self.tag): raise exceptions.SyntaxException( - "Closing tag without opening tag: " % - match.group(1), - **self.exception_kwargs) + "Closing tag without opening tag: " + % match.group(1), + **self.exception_kwargs + ) elif self.tag[-1].keyword != match.group(1): raise exceptions.SyntaxException( - "Closing tag does not match tag: <%%%s>" % - (match.group(1), self.tag[-1].keyword), - **self.exception_kwargs) + "Closing tag does not match tag: <%%%s>" + % (match.group(1), self.tag[-1].keyword), + **self.exception_kwargs + ) self.tag.pop() return True else: return False def match_end(self): - match = self.match(r'\Z', re.S) + match = self.match(r"\Z", re.S) if match: string = match.group() if string: @@ -345,7 +373,8 @@ class Lexer(object): return False def match_text(self): - match = self.match(r""" + match = self.match( + r""" (.*?) # anything, followed by: ( (?<=\n)(?=[ \t]*(?=%|\#\#)) # an eval or line-based @@ -360,7 +389,9 @@ class Lexer(object): (\\\r?\n) # an escaped newline - throw away | \Z # end of string - )""", re.X | re.S) + )""", + re.X | re.S, + ) if match: text = match.group(1) @@ -374,14 +405,17 @@ class Lexer(object): match = self.match(r"<%(!)?") if match: line, pos = self.matched_lineno, self.matched_charpos - text, end = self.parse_until_text(False, r'%>') + text, end = self.parse_until_text(False, r"%>") # the trailing newline helps # compiler.parse() not complain about indentation text = adjust_whitespace(text) + "\n" self.append_node( parsetree.Code, text, - match.group(1) == '!', lineno=line, pos=pos) + match.group(1) == "!", + lineno=line, + pos=pos, + ) return True else: return False @@ -390,16 +424,19 @@ class Lexer(object): match = self.match(r"\${") if match: line, pos = self.matched_lineno, self.matched_charpos - text, end = self.parse_until_text(True, r'\|', r'}') - if end == '|': - escapes, end = self.parse_until_text(True, r'}') + text, end = self.parse_until_text(True, r"\|", r"}") + if end == "|": + escapes, end = self.parse_until_text(True, r"}") else: escapes = "" - text = text.replace('\r\n', '\n') + text = text.replace("\r\n", "\n") self.append_node( parsetree.Expression, - text, escapes.strip(), - lineno=line, pos=pos) + text, + escapes.strip(), + lineno=line, + pos=pos, + ) return True else: return False @@ -407,31 +444,35 @@ class Lexer(object): def match_control_line(self): match = self.match( r"(?<=^)[\t ]*(%(?!%)|##)[\t ]*((?:(?:\\r?\n)|[^\r\n])*)" - r"(?:\r?\n|\Z)", re.M) + r"(?:\r?\n|\Z)", + re.M, + ) if match: operator = match.group(1) text = match.group(2) - if operator == '%': - m2 = re.match(r'(end)?(\w+)\s*(.*)', text) + if operator == "%": + m2 = re.match(r"(end)?(\w+)\s*(.*)", text) if not m2: raise exceptions.SyntaxException( - "Invalid control line: '%s'" % - text, - **self.exception_kwargs) + "Invalid control line: '%s'" % text, + **self.exception_kwargs + ) isend, keyword = m2.group(1, 2) - isend = (isend is not None) + isend = isend is not None if isend: if not len(self.control_line): raise exceptions.SyntaxException( - "No starting keyword '%s' for '%s'" % - (keyword, text), - **self.exception_kwargs) + "No starting keyword '%s' for '%s'" + % (keyword, text), + **self.exception_kwargs + ) elif self.control_line[-1].keyword != keyword: raise exceptions.SyntaxException( - "Keyword '%s' doesn't match keyword '%s'" % - (text, self.control_line[-1].keyword), - **self.exception_kwargs) + "Keyword '%s' doesn't match keyword '%s'" + % (text, self.control_line[-1].keyword), + **self.exception_kwargs + ) self.append_node(parsetree.ControlLine, keyword, isend, text) else: self.append_node(parsetree.Comment, text) diff --git a/server/www/packages/packages-linux/x64/mako/lookup.py b/server/www/packages/packages-linux/x64/mako/lookup.py index 0d3f304..93558b2 100644 --- a/server/www/packages/packages-linux/x64/mako/lookup.py +++ b/server/www/packages/packages-linux/x64/mako/lookup.py @@ -1,14 +1,16 @@ # mako/lookup.py -# Copyright (C) 2006-2016 the Mako authors and contributors +# Copyright 2006-2019 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php import os -import stat import posixpath import re -from mako import exceptions, util +import stat + +from mako import exceptions +from mako import util from mako.template import Template try: @@ -151,41 +153,41 @@ class TemplateLookup(TemplateCollection): """ - def __init__(self, - directories=None, - module_directory=None, - filesystem_checks=True, - collection_size=-1, - format_exceptions=False, - error_handler=None, - disable_unicode=False, - bytestring_passthrough=False, - output_encoding=None, - encoding_errors='strict', + def __init__( + self, + directories=None, + module_directory=None, + filesystem_checks=True, + collection_size=-1, + format_exceptions=False, + error_handler=None, + disable_unicode=False, + bytestring_passthrough=False, + output_encoding=None, + encoding_errors="strict", + cache_args=None, + cache_impl="beaker", + cache_enabled=True, + cache_type=None, + cache_dir=None, + cache_url=None, + modulename_callable=None, + module_writer=None, + default_filters=None, + buffer_filters=(), + strict_undefined=False, + imports=None, + future_imports=None, + enable_loop=True, + input_encoding=None, + preprocessor=None, + lexer_cls=None, + include_error_handler=None, + ): - cache_args=None, - cache_impl='beaker', - cache_enabled=True, - cache_type=None, - cache_dir=None, - cache_url=None, - - modulename_callable=None, - module_writer=None, - default_filters=None, - buffer_filters=(), - strict_undefined=False, - imports=None, - future_imports=None, - enable_loop=True, - input_encoding=None, - preprocessor=None, - lexer_cls=None, - include_error_handler=None): - - self.directories = [posixpath.normpath(d) for d in - util.to_list(directories, ()) - ] + self.directories = [ + posixpath.normpath(d) for d in util.to_list(directories, ()) + ] self.module_directory = module_directory self.modulename_callable = modulename_callable self.filesystem_checks = filesystem_checks @@ -195,34 +197,34 @@ class TemplateLookup(TemplateCollection): cache_args = {} # transfer deprecated cache_* args if cache_dir: - cache_args.setdefault('dir', cache_dir) + cache_args.setdefault("dir", cache_dir) if cache_url: - cache_args.setdefault('url', cache_url) + cache_args.setdefault("url", cache_url) if cache_type: - cache_args.setdefault('type', cache_type) + cache_args.setdefault("type", cache_type) self.template_args = { - 'format_exceptions': format_exceptions, - 'error_handler': error_handler, - 'include_error_handler': include_error_handler, - 'disable_unicode': disable_unicode, - 'bytestring_passthrough': bytestring_passthrough, - 'output_encoding': output_encoding, - 'cache_impl': cache_impl, - 'encoding_errors': encoding_errors, - 'input_encoding': input_encoding, - 'module_directory': module_directory, - 'module_writer': module_writer, - 'cache_args': cache_args, - 'cache_enabled': cache_enabled, - 'default_filters': default_filters, - 'buffer_filters': buffer_filters, - 'strict_undefined': strict_undefined, - 'imports': imports, - 'future_imports': future_imports, - 'enable_loop': enable_loop, - 'preprocessor': preprocessor, - 'lexer_cls': lexer_cls + "format_exceptions": format_exceptions, + "error_handler": error_handler, + "include_error_handler": include_error_handler, + "disable_unicode": disable_unicode, + "bytestring_passthrough": bytestring_passthrough, + "output_encoding": output_encoding, + "cache_impl": cache_impl, + "encoding_errors": encoding_errors, + "input_encoding": input_encoding, + "module_directory": module_directory, + "module_writer": module_writer, + "cache_args": cache_args, + "cache_enabled": cache_enabled, + "default_filters": default_filters, + "buffer_filters": buffer_filters, + "strict_undefined": strict_undefined, + "imports": imports, + "future_imports": future_imports, + "enable_loop": enable_loop, + "preprocessor": preprocessor, + "lexer_cls": lexer_cls, } if collection_size == -1: @@ -248,17 +250,18 @@ class TemplateLookup(TemplateCollection): else: return self._collection[uri] except KeyError: - u = re.sub(r'^\/+', '', uri) - for dir in self.directories: + u = re.sub(r"^\/+", "", uri) + for dir_ in self.directories: # make sure the path seperators are posix - os.altsep is empty # on POSIX and cannot be used. - dir = dir.replace(os.path.sep, posixpath.sep) - srcfile = posixpath.normpath(posixpath.join(dir, u)) + dir_ = dir_.replace(os.path.sep, posixpath.sep) + srcfile = posixpath.normpath(posixpath.join(dir_, u)) if os.path.isfile(srcfile): return self._load(srcfile, uri) else: raise exceptions.TopLevelLookupException( - "Cant locate template for uri %r" % uri) + "Cant locate template for uri %r" % uri + ) def adjust_uri(self, uri, relativeto): """Adjust the given ``uri`` based on the given relative URI.""" @@ -267,12 +270,13 @@ class TemplateLookup(TemplateCollection): if key in self._uri_cache: return self._uri_cache[key] - if uri[0] != '/': + if uri[0] != "/": if relativeto is not None: v = self._uri_cache[key] = posixpath.join( - posixpath.dirname(relativeto), uri) + posixpath.dirname(relativeto), uri + ) else: - v = self._uri_cache[key] = '/' + uri + v = self._uri_cache[key] = "/" + uri else: v = self._uri_cache[key] = uri return v @@ -295,9 +299,9 @@ class TemplateLookup(TemplateCollection): """ filename = posixpath.normpath(filename) - for dir in self.directories: - if filename[0:len(dir)] == dir: - return filename[len(dir):] + for dir_ in self.directories: + if filename[0 : len(dir_)] == dir_: + return filename[len(dir_) :] else: return None @@ -320,7 +324,8 @@ class TemplateLookup(TemplateCollection): filename=posixpath.normpath(filename), lookup=self, module_filename=module_filename, - **self.template_args) + **self.template_args + ) return template except: # if compilation fails etc, ensure @@ -337,8 +342,7 @@ class TemplateLookup(TemplateCollection): try: template_stat = os.stat(template.filename) - if template.module._modified_time < \ - template_stat[stat.ST_MTIME]: + if template.module._modified_time < template_stat[stat.ST_MTIME]: self._collection.pop(uri, None) return self._load(template.filename, uri) else: @@ -346,7 +350,8 @@ class TemplateLookup(TemplateCollection): except OSError: self._collection.pop(uri, None) raise exceptions.TemplateLookupException( - "Cant locate template for uri %r" % uri) + "Cant locate template for uri %r" % uri + ) def put_string(self, uri, text): """Place a new :class:`.Template` object into this @@ -355,10 +360,8 @@ class TemplateLookup(TemplateCollection): """ self._collection[uri] = Template( - text, - lookup=self, - uri=uri, - **self.template_args) + text, lookup=self, uri=uri, **self.template_args + ) def put_template(self, uri, template): """Place a new :class:`.Template` object into this diff --git a/server/www/packages/packages-linux/x64/mako/parsetree.py b/server/www/packages/packages-linux/x64/mako/parsetree.py index e129916..2881da1 100644 --- a/server/www/packages/packages-linux/x64/mako/parsetree.py +++ b/server/www/packages/packages-linux/x64/mako/parsetree.py @@ -1,14 +1,19 @@ # mako/parsetree.py -# Copyright (C) 2006-2016 the Mako authors and contributors +# Copyright 2006-2019 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php """defines the parse tree components for Mako templates.""" -from mako import exceptions, ast, util, filters, compat import re +from mako import ast +from mako import compat +from mako import exceptions +from mako import filters +from mako import util + class Node(object): @@ -22,8 +27,12 @@ class Node(object): @property def exception_kwargs(self): - return {'source': self.source, 'lineno': self.lineno, - 'pos': self.pos, 'filename': self.filename} + return { + "source": self.source, + "lineno": self.lineno, + "pos": self.pos, + "filename": self.filename, + } def get_children(self): return [] @@ -42,7 +51,7 @@ class TemplateNode(Node): """a 'container' node that stores the overall collection of nodes.""" def __init__(self, filename): - super(TemplateNode, self).__init__('', 0, 0, filename) + super(TemplateNode, self).__init__("", 0, 0, filename) self.nodes = [] self.page_attributes = {} @@ -52,7 +61,8 @@ class TemplateNode(Node): def __repr__(self): return "TemplateNode(%s, %r)" % ( util.sorted_dict_repr(self.page_attributes), - self.nodes) + self.nodes, + ) class ControlLine(Node): @@ -74,7 +84,7 @@ class ControlLine(Node): self.text = text self.keyword = keyword self.isend = isend - self.is_primary = keyword in ['for', 'if', 'while', 'try', 'with'] + self.is_primary = keyword in ["for", "if", "while", "try", "with"] self.nodes = [] if self.isend: self._declared_identifiers = [] @@ -98,9 +108,9 @@ class ControlLine(Node): for this ControlLine""" return keyword in { - 'if': set(['else', 'elif']), - 'try': set(['except', 'finally']), - 'for': set(['else']) + "if": set(["else", "elif"]), + "try": set(["except", "finally"]), + "for": set(["else"]), }.get(self.keyword, []) def __repr__(self): @@ -108,7 +118,7 @@ class ControlLine(Node): self.keyword, self.text, self.isend, - (self.lineno, self.pos) + (self.lineno, self.pos), ) @@ -158,7 +168,7 @@ class Code(Node): return "Code(%r, %r, %r)" % ( self.text, self.ismodule, - (self.lineno, self.pos) + (self.lineno, self.pos), ) @@ -208,7 +218,7 @@ class Expression(Node): return "Expression(%r, %r, %r)" % ( self.text, self.escapes_code.args, - (self.lineno, self.pos) + (self.lineno, self.pos), ) @@ -219,45 +229,55 @@ class _TagMeta(type): _classmap = {} - def __init__(cls, clsname, bases, dict): - if getattr(cls, '__keyword__', None) is not None: + def __init__(cls, clsname, bases, dict_): + if getattr(cls, "__keyword__", None) is not None: cls._classmap[cls.__keyword__] = cls - super(_TagMeta, cls).__init__(clsname, bases, dict) + super(_TagMeta, cls).__init__(clsname, bases, dict_) def __call__(cls, keyword, attributes, **kwargs): if ":" in keyword: - ns, defname = keyword.split(':') - return type.__call__(CallNamespaceTag, ns, defname, - attributes, **kwargs) + ns, defname = keyword.split(":") + return type.__call__( + CallNamespaceTag, ns, defname, attributes, **kwargs + ) try: cls = _TagMeta._classmap[keyword] except KeyError: raise exceptions.CompileException( "No such tag: '%s'" % keyword, - source=kwargs['source'], - lineno=kwargs['lineno'], - pos=kwargs['pos'], - filename=kwargs['filename'] + source=kwargs["source"], + lineno=kwargs["lineno"], + pos=kwargs["pos"], + filename=kwargs["filename"], ) return type.__call__(cls, keyword, attributes, **kwargs) class Tag(compat.with_metaclass(_TagMeta, Node)): - """abstract base class for tags. - <%sometag/> + e.g.:: - <%someothertag> - stuff - + <%sometag/> + + <%someothertag> + stuff + """ + __keyword__ = None - def __init__(self, keyword, attributes, expressions, - nonexpressions, required, **kwargs): + def __init__( + self, + keyword, + attributes, + expressions, + nonexpressions, + required, + **kwargs + ): r"""construct a new Tag instance. this constructor not called directly, and is only called @@ -284,9 +304,10 @@ class Tag(compat.with_metaclass(_TagMeta, Node)): missing = [r for r in required if r not in self.parsed_attributes] if len(missing): raise exceptions.CompileException( - "Missing attribute(s): %s" % - ",".join([repr(m) for m in missing]), - **self.exception_kwargs) + "Missing attribute(s): %s" + % ",".join([repr(m) for m in missing]), + **self.exception_kwargs + ) self.parent = None self.nodes = [] @@ -302,36 +323,40 @@ class Tag(compat.with_metaclass(_TagMeta, Node)): for key in self.attributes: if key in expressions: expr = [] - for x in re.compile(r'(\${.+?})', - re.S).split(self.attributes[key]): - m = re.compile(r'^\${(.+?)}$', re.S).match(x) + for x in re.compile(r"(\${.+?})", re.S).split( + self.attributes[key] + ): + m = re.compile(r"^\${(.+?)}$", re.S).match(x) if m: - code = ast.PythonCode(m.group(1).rstrip(), - **self.exception_kwargs) + code = ast.PythonCode( + m.group(1).rstrip(), **self.exception_kwargs + ) # we aren't discarding "declared_identifiers" here, # which we do so that list comprehension-declared # variables aren't counted. As yet can't find a # condition that requires it here. - undeclared_identifiers = \ - undeclared_identifiers.union( - code.undeclared_identifiers) - expr.append('(%s)' % m.group(1)) + undeclared_identifiers = undeclared_identifiers.union( + code.undeclared_identifiers + ) + expr.append("(%s)" % m.group(1)) else: if x: expr.append(repr(x)) - self.parsed_attributes[key] = " + ".join(expr) or repr('') + self.parsed_attributes[key] = " + ".join(expr) or repr("") elif key in nonexpressions: - if re.search(r'\${.+?}', self.attributes[key]): + if re.search(r"\${.+?}", self.attributes[key]): raise exceptions.CompileException( "Attibute '%s' in tag '%s' does not allow embedded " "expressions" % (key, self.keyword), - **self.exception_kwargs) + **self.exception_kwargs + ) self.parsed_attributes[key] = repr(self.attributes[key]) else: raise exceptions.CompileException( - "Invalid attribute for tag '%s': '%s'" % - (self.keyword, key), - **self.exception_kwargs) + "Invalid attribute for tag '%s': '%s'" + % (self.keyword, key), + **self.exception_kwargs + ) self.expression_undeclared_identifiers = undeclared_identifiers def declared_identifiers(self): @@ -341,56 +366,64 @@ class Tag(compat.with_metaclass(_TagMeta, Node)): return self.expression_undeclared_identifiers def __repr__(self): - return "%s(%r, %s, %r, %r)" % (self.__class__.__name__, - self.keyword, - util.sorted_dict_repr(self.attributes), - (self.lineno, self.pos), - self.nodes - ) + return "%s(%r, %s, %r, %r)" % ( + self.__class__.__name__, + self.keyword, + util.sorted_dict_repr(self.attributes), + (self.lineno, self.pos), + self.nodes, + ) class IncludeTag(Tag): - __keyword__ = 'include' + __keyword__ = "include" def __init__(self, keyword, attributes, **kwargs): super(IncludeTag, self).__init__( keyword, attributes, - ('file', 'import', 'args'), - (), ('file',), **kwargs) + ("file", "import", "args"), + (), + ("file",), + **kwargs + ) self.page_args = ast.PythonCode( - "__DUMMY(%s)" % attributes.get('args', ''), - **self.exception_kwargs) + "__DUMMY(%s)" % attributes.get("args", ""), **self.exception_kwargs + ) def declared_identifiers(self): return [] def undeclared_identifiers(self): - identifiers = self.page_args.undeclared_identifiers.\ - difference(set(["__DUMMY"])).\ - difference(self.page_args.declared_identifiers) - return identifiers.union(super(IncludeTag, self). - undeclared_identifiers()) + identifiers = self.page_args.undeclared_identifiers.difference( + set(["__DUMMY"]) + ).difference(self.page_args.declared_identifiers) + return identifiers.union( + super(IncludeTag, self).undeclared_identifiers() + ) class NamespaceTag(Tag): - __keyword__ = 'namespace' + __keyword__ = "namespace" def __init__(self, keyword, attributes, **kwargs): super(NamespaceTag, self).__init__( - keyword, attributes, - ('file',), - ('name', 'inheritable', - 'import', 'module'), - (), **kwargs) + keyword, + attributes, + ("file",), + ("name", "inheritable", "import", "module"), + (), + **kwargs + ) - self.name = attributes.get('name', '__anon_%s' % hex(abs(id(self)))) - if 'name' not in attributes and 'import' not in attributes: + self.name = attributes.get("name", "__anon_%s" % hex(abs(id(self)))) + if "name" not in attributes and "import" not in attributes: raise exceptions.CompileException( "'name' and/or 'import' attributes are required " "for <%namespace>", - **self.exception_kwargs) - if 'file' in attributes and 'module' in attributes: + **self.exception_kwargs + ) + if "file" in attributes and "module" in attributes: raise exceptions.CompileException( "<%namespace> may only have one of 'file' or 'module'", **self.exception_kwargs @@ -401,51 +434,51 @@ class NamespaceTag(Tag): class TextTag(Tag): - __keyword__ = 'text' + __keyword__ = "text" def __init__(self, keyword, attributes, **kwargs): super(TextTag, self).__init__( - keyword, - attributes, (), - ('filter'), (), **kwargs) + keyword, attributes, (), ("filter"), (), **kwargs + ) self.filter_args = ast.ArgumentList( - attributes.get('filter', ''), - **self.exception_kwargs) + attributes.get("filter", ""), **self.exception_kwargs + ) def undeclared_identifiers(self): - return self.filter_args.\ - undeclared_identifiers.\ - difference(filters.DEFAULT_ESCAPES.keys()).union( - self.expression_undeclared_identifiers - ) + return self.filter_args.undeclared_identifiers.difference( + filters.DEFAULT_ESCAPES.keys() + ).union(self.expression_undeclared_identifiers) class DefTag(Tag): - __keyword__ = 'def' + __keyword__ = "def" def __init__(self, keyword, attributes, **kwargs): - expressions = ['buffered', 'cached'] + [ - c for c in attributes if c.startswith('cache_')] + expressions = ["buffered", "cached"] + [ + c for c in attributes if c.startswith("cache_") + ] super(DefTag, self).__init__( keyword, attributes, expressions, - ('name', 'filter', 'decorator'), - ('name',), - **kwargs) - name = attributes['name'] - if re.match(r'^[\w_]+$', name): + ("name", "filter", "decorator"), + ("name",), + **kwargs + ) + name = attributes["name"] + if re.match(r"^[\w_]+$", name): raise exceptions.CompileException( - "Missing parenthesis in %def", - **self.exception_kwargs) - self.function_decl = ast.FunctionDecl("def " + name + ":pass", - **self.exception_kwargs) + "Missing parenthesis in %def", **self.exception_kwargs + ) + self.function_decl = ast.FunctionDecl( + "def " + name + ":pass", **self.exception_kwargs + ) self.name = self.function_decl.funcname - self.decorator = attributes.get('decorator', '') + self.decorator = attributes.get("decorator", "") self.filter_args = ast.ArgumentList( - attributes.get('filter', ''), - **self.exception_kwargs) + attributes.get("filter", ""), **self.exception_kwargs + ) is_anonymous = False is_block = False @@ -463,51 +496,58 @@ class DefTag(Tag): def undeclared_identifiers(self): res = [] for c in self.function_decl.defaults: - res += list(ast.PythonCode(c, **self.exception_kwargs). - undeclared_identifiers) - return set(res).union( - self.filter_args. - undeclared_identifiers. - difference(filters.DEFAULT_ESCAPES.keys()) - ).union( - self.expression_undeclared_identifiers - ).difference( - self.function_decl.allargnames + res += list( + ast.PythonCode( + c, **self.exception_kwargs + ).undeclared_identifiers + ) + return ( + set(res) + .union( + self.filter_args.undeclared_identifiers.difference( + filters.DEFAULT_ESCAPES.keys() + ) + ) + .union(self.expression_undeclared_identifiers) + .difference(self.function_decl.allargnames) ) class BlockTag(Tag): - __keyword__ = 'block' + __keyword__ = "block" def __init__(self, keyword, attributes, **kwargs): - expressions = ['buffered', 'cached', 'args'] + [ - c for c in attributes if c.startswith('cache_')] + expressions = ["buffered", "cached", "args"] + [ + c for c in attributes if c.startswith("cache_") + ] super(BlockTag, self).__init__( keyword, attributes, expressions, - ('name', 'filter', 'decorator'), + ("name", "filter", "decorator"), (), - **kwargs) - name = attributes.get('name') - if name and not re.match(r'^[\w_]+$', name): + **kwargs + ) + name = attributes.get("name") + if name and not re.match(r"^[\w_]+$", name): raise exceptions.CompileException( "%block may not specify an argument signature", - **self.exception_kwargs) - if not name and attributes.get('args', None): - raise exceptions.CompileException( - "Only named %blocks may specify args", **self.exception_kwargs ) - self.body_decl = ast.FunctionArgs(attributes.get('args', ''), - **self.exception_kwargs) + if not name and attributes.get("args", None): + raise exceptions.CompileException( + "Only named %blocks may specify args", **self.exception_kwargs + ) + self.body_decl = ast.FunctionArgs( + attributes.get("args", ""), **self.exception_kwargs + ) self.name = name - self.decorator = attributes.get('decorator', '') + self.decorator = attributes.get("decorator", "") self.filter_args = ast.ArgumentList( - attributes.get('filter', ''), - **self.exception_kwargs) + attributes.get("filter", ""), **self.exception_kwargs + ) is_block = True @@ -517,7 +557,7 @@ class BlockTag(Tag): @property def funcname(self): - return self.name or "__M_anon_%d" % (self.lineno, ) + return self.name or "__M_anon_%d" % (self.lineno,) def get_argument_expressions(self, **kw): return self.body_decl.get_argument_expressions(**kw) @@ -526,91 +566,100 @@ class BlockTag(Tag): return self.body_decl.allargnames def undeclared_identifiers(self): - return (self.filter_args. - undeclared_identifiers. - difference(filters.DEFAULT_ESCAPES.keys()) - ).union(self.expression_undeclared_identifiers) + return ( + self.filter_args.undeclared_identifiers.difference( + filters.DEFAULT_ESCAPES.keys() + ) + ).union(self.expression_undeclared_identifiers) class CallTag(Tag): - __keyword__ = 'call' + __keyword__ = "call" def __init__(self, keyword, attributes, **kwargs): - super(CallTag, self).__init__(keyword, attributes, - ('args'), ('expr',), ('expr',), **kwargs) - self.expression = attributes['expr'] + super(CallTag, self).__init__( + keyword, attributes, ("args"), ("expr",), ("expr",), **kwargs + ) + self.expression = attributes["expr"] self.code = ast.PythonCode(self.expression, **self.exception_kwargs) - self.body_decl = ast.FunctionArgs(attributes.get('args', ''), - **self.exception_kwargs) + self.body_decl = ast.FunctionArgs( + attributes.get("args", ""), **self.exception_kwargs + ) def declared_identifiers(self): return self.code.declared_identifiers.union(self.body_decl.allargnames) def undeclared_identifiers(self): - return self.code.undeclared_identifiers.\ - difference(self.code.declared_identifiers) + return self.code.undeclared_identifiers.difference( + self.code.declared_identifiers + ) class CallNamespaceTag(Tag): - def __init__(self, namespace, defname, attributes, **kwargs): super(CallNamespaceTag, self).__init__( namespace + ":" + defname, attributes, - tuple(attributes.keys()) + ('args', ), + tuple(attributes.keys()) + ("args",), (), (), - **kwargs) + **kwargs + ) self.expression = "%s.%s(%s)" % ( namespace, defname, - ",".join(["%s=%s" % (k, v) for k, v in - self.parsed_attributes.items() - if k != 'args']) + ",".join( + [ + "%s=%s" % (k, v) + for k, v in self.parsed_attributes.items() + if k != "args" + ] + ), ) self.code = ast.PythonCode(self.expression, **self.exception_kwargs) self.body_decl = ast.FunctionArgs( - attributes.get('args', ''), - **self.exception_kwargs) + attributes.get("args", ""), **self.exception_kwargs + ) def declared_identifiers(self): return self.code.declared_identifiers.union(self.body_decl.allargnames) def undeclared_identifiers(self): - return self.code.undeclared_identifiers.\ - difference(self.code.declared_identifiers) + return self.code.undeclared_identifiers.difference( + self.code.declared_identifiers + ) class InheritTag(Tag): - __keyword__ = 'inherit' + __keyword__ = "inherit" def __init__(self, keyword, attributes, **kwargs): super(InheritTag, self).__init__( - keyword, attributes, - ('file',), (), ('file',), **kwargs) + keyword, attributes, ("file",), (), ("file",), **kwargs + ) class PageTag(Tag): - __keyword__ = 'page' + __keyword__ = "page" def __init__(self, keyword, attributes, **kwargs): - expressions = \ - ['cached', 'args', 'expression_filter', 'enable_loop'] + \ - [c for c in attributes if c.startswith('cache_')] + expressions = [ + "cached", + "args", + "expression_filter", + "enable_loop", + ] + [c for c in attributes if c.startswith("cache_")] super(PageTag, self).__init__( - keyword, - attributes, - expressions, - (), - (), - **kwargs) - self.body_decl = ast.FunctionArgs(attributes.get('args', ''), - **self.exception_kwargs) + keyword, attributes, expressions, (), (), **kwargs + ) + self.body_decl = ast.FunctionArgs( + attributes.get("args", ""), **self.exception_kwargs + ) self.filter_args = ast.ArgumentList( - attributes.get('expression_filter', ''), - **self.exception_kwargs) + attributes.get("expression_filter", ""), **self.exception_kwargs + ) def declared_identifiers(self): return self.body_decl.allargnames diff --git a/server/www/packages/packages-linux/x64/mako/pygen.py b/server/www/packages/packages-linux/x64/mako/pygen.py index 8514e02..603676d 100644 --- a/server/www/packages/packages-linux/x64/mako/pygen.py +++ b/server/www/packages/packages-linux/x64/mako/pygen.py @@ -1,5 +1,5 @@ # mako/pygen.py -# Copyright (C) 2006-2016 the Mako authors and contributors +# Copyright 2006-2019 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -7,11 +7,11 @@ """utilities for generating and formatting literal Python code.""" import re + from mako import exceptions class PythonPrinter(object): - def __init__(self, stream): # indentation counter self.indent = 0 @@ -54,14 +54,16 @@ class PythonPrinter(object): self.stream.write("\n" * num) self._update_lineno(num) - def write_indented_block(self, block): + def write_indented_block(self, block, starting_lineno=None): """print a line or lines of python which already contain indentation. The indentation of the total block of lines will be adjusted to that of the current indent level.""" self.in_indent_lines = False - for l in re.split(r'\r?\n', block): + for i, l in enumerate(re.split(r"\r?\n", block)): self.line_buffer.append(l) + if starting_lineno is not None: + self.start_source(starting_lineno + i) self._update_lineno(1) def writelines(self, *lines): @@ -83,21 +85,18 @@ class PythonPrinter(object): self.in_indent_lines = True if ( - line is None or - re.match(r"^\s*#", line) or - re.match(r"^\s*$", line) + line is None + or re.match(r"^\s*#", line) + or re.match(r"^\s*$", line) ): hastext = False else: hastext = True - is_comment = line and len(line) and line[0] == '#' + is_comment = line and len(line) and line[0] == "#" # see if this line should decrease the indentation level - if ( - not is_comment and - (not hastext or self._is_unindentor(line)) - ): + if not is_comment and (not hastext or self._is_unindentor(line)): if self.indent > 0: self.indent -= 1 @@ -106,7 +105,8 @@ class PythonPrinter(object): # module wont compile. if len(self.indent_detail) == 0: raise exceptions.SyntaxException( - "Too many whitespace closures") + "Too many whitespace closures" + ) self.indent_detail.pop() if line is None: @@ -136,8 +136,9 @@ class PythonPrinter(object): # its not a "compound" keyword. but lets also # test for valid Python keywords that might be indenting us, # else assume its a non-indenting line - m2 = re.match(r"^\s*(def|class|else|elif|except|finally)", - line) + m2 = re.match( + r"^\s*(def|class|else|elif|except|finally)", line + ) if m2: self.indent += 1 self.indent_detail.append(indentor) @@ -189,14 +190,15 @@ class PythonPrinter(object): # return False - def _indent_line(self, line, stripspace=''): + def _indent_line(self, line, stripspace=""): """indent the given line according to the current indent level. stripspace is a string of space that will be truncated from the start of the line before indenting.""" - return re.sub(r"^%s" % stripspace, self.indentstring - * self.indent, line) + return re.sub( + r"^%s" % stripspace, self.indentstring * self.indent, line + ) def _reset_multi_line_flags(self): """reset the flags which would indicate we are in a backslashed @@ -214,7 +216,7 @@ class PythonPrinter(object): # a literal multiline string with unfortunately placed # whitespace - current_state = (self.backslashed or self.triplequoted) + current_state = self.backslashed or self.triplequoted if re.search(r"\\$", line): self.backslashed = True @@ -251,7 +253,7 @@ def adjust_whitespace(text): (backslashed, triplequoted) = (0, 1) def in_multi_line(line): - start_state = (state[backslashed] or state[triplequoted]) + start_state = state[backslashed] or state[triplequoted] if re.search(r"\\$", line): state[backslashed] = True @@ -261,7 +263,7 @@ def adjust_whitespace(text): def match(reg, t): m = re.match(reg, t) if m: - return m, t[len(m.group(0)):] + return m, t[len(m.group(0)) :] else: return None, t @@ -273,7 +275,7 @@ def adjust_whitespace(text): else: m, line = match(r".*?(?=%s|$)" % state[triplequoted], line) else: - m, line = match(r'#', line) + m, line = match(r"#", line) if m: return start_state @@ -286,13 +288,13 @@ def adjust_whitespace(text): return start_state - def _indent_line(line, stripspace=''): - return re.sub(r"^%s" % stripspace, '', line) + def _indent_line(line, stripspace=""): + return re.sub(r"^%s" % stripspace, "", line) lines = [] stripspace = None - for line in re.split(r'\r?\n', text): + for line in re.split(r"\r?\n", text): if in_multi_line(line): lines.append(line) else: diff --git a/server/www/packages/packages-linux/x64/mako/pyparser.py b/server/www/packages/packages-linux/x64/mako/pyparser.py index 15d0da6..e41c304 100644 --- a/server/www/packages/packages-linux/x64/mako/pyparser.py +++ b/server/www/packages/packages-linux/x64/mako/pyparser.py @@ -1,5 +1,5 @@ # mako/pyparser.py -# Copyright (C) 2006-2016 the Mako authors and contributors +# Copyright 2006-2019 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -10,46 +10,52 @@ Parsing to AST is done via _ast on Python > 2.5, otherwise the compiler module is used. """ -from mako import exceptions, util, compat -from mako.compat import arg_stringname import operator +import _ast + +from mako import _ast_util +from mako import compat +from mako import exceptions +from mako import util +from mako.compat import arg_stringname + if compat.py3k: # words that cannot be assigned to (notably # smaller than the total keys in __builtins__) - reserved = set(['True', 'False', 'None', 'print']) + reserved = set(["True", "False", "None", "print"]) # the "id" attribute on a function node - arg_id = operator.attrgetter('arg') + arg_id = operator.attrgetter("arg") else: # words that cannot be assigned to (notably # smaller than the total keys in __builtins__) - reserved = set(['True', 'False', 'None']) + reserved = set(["True", "False", "None"]) # the "id" attribute on a function node - arg_id = operator.attrgetter('id') + arg_id = operator.attrgetter("id") -import _ast util.restore__ast(_ast) -from mako import _ast_util -def parse(code, mode='exec', **exception_kwargs): +def parse(code, mode="exec", **exception_kwargs): """Parse an expression into AST""" try: - return _ast_util.parse(code, '', mode) + return _ast_util.parse(code, "", mode) except Exception: raise exceptions.SyntaxException( - "(%s) %s (%r)" % ( + "(%s) %s (%r)" + % ( compat.exception_as().__class__.__name__, compat.exception_as(), - code[0:50] - ), **exception_kwargs) + code[0:50], + ), + **exception_kwargs + ) class FindIdentifiers(_ast_util.NodeVisitor): - def __init__(self, listener, **exception_kwargs): self.in_function = False self.in_assign_targets = False @@ -119,9 +125,9 @@ class FindIdentifiers(_ast_util.NodeVisitor): self.in_function = True local_ident_stack = self.local_ident_stack - self.local_ident_stack = local_ident_stack.union([ - arg_id(arg) for arg in self._expand_tuples(node.args.args) - ]) + self.local_ident_stack = local_ident_stack.union( + [arg_id(arg) for arg in self._expand_tuples(node.args.args)] + ) if islambda: self.visit(node.body) else: @@ -146,9 +152,11 @@ class FindIdentifiers(_ast_util.NodeVisitor): # this is eqiuvalent to visit_AssName in # compiler self._add_declared(node.id) - elif node.id not in reserved and node.id \ - not in self.listener.declared_identifiers and node.id \ - not in self.local_ident_stack: + elif ( + node.id not in reserved + and node.id not in self.listener.declared_identifiers + and node.id not in self.local_ident_stack + ): self.listener.undeclared_identifiers.add(node.id) def visit_Import(self, node): @@ -156,24 +164,25 @@ class FindIdentifiers(_ast_util.NodeVisitor): if name.asname is not None: self._add_declared(name.asname) else: - self._add_declared(name.name.split('.')[0]) + self._add_declared(name.name.split(".")[0]) def visit_ImportFrom(self, node): for name in node.names: if name.asname is not None: self._add_declared(name.asname) else: - if name.name == '*': + if name.name == "*": raise exceptions.CompileException( "'import *' is not supported, since all identifier " "names must be explicitly declared. Please use the " "form 'from import , , " - "...' instead.", **self.exception_kwargs) + "...' instead.", + **self.exception_kwargs + ) self._add_declared(name.name) class FindTuple(_ast_util.NodeVisitor): - def __init__(self, listener, code_factory, **exception_kwargs): self.listener = listener self.exception_kwargs = exception_kwargs @@ -184,16 +193,17 @@ class FindTuple(_ast_util.NodeVisitor): p = self.code_factory(n, **self.exception_kwargs) self.listener.codeargs.append(p) self.listener.args.append(ExpressionGenerator(n).value()) - self.listener.declared_identifiers = \ - self.listener.declared_identifiers.union( - p.declared_identifiers) - self.listener.undeclared_identifiers = \ - self.listener.undeclared_identifiers.union( - p.undeclared_identifiers) + ldi = self.listener.declared_identifiers + self.listener.declared_identifiers = ldi.union( + p.declared_identifiers + ) + lui = self.listener.undeclared_identifiers + self.listener.undeclared_identifiers = lui.union( + p.undeclared_identifiers + ) class ParseFunc(_ast_util.NodeVisitor): - def __init__(self, listener, **exception_kwargs): self.listener = listener self.exception_kwargs = exception_kwargs @@ -224,10 +234,9 @@ class ParseFunc(_ast_util.NodeVisitor): class ExpressionGenerator(object): - def __init__(self, astnode): - self.generator = _ast_util.SourceGenerator(' ' * 4) + self.generator = _ast_util.SourceGenerator(" " * 4) self.generator.visit(astnode) def value(self): - return ''.join(self.generator.result) + return "".join(self.generator.result) diff --git a/server/www/packages/packages-linux/x64/mako/runtime.py b/server/www/packages/packages-linux/x64/mako/runtime.py index 769541c..0e7149b 100644 --- a/server/www/packages/packages-linux/x64/mako/runtime.py +++ b/server/www/packages/packages-linux/x64/mako/runtime.py @@ -1,5 +1,5 @@ # mako/runtime.py -# Copyright (C) 2006-2016 the Mako authors and contributors +# Copyright 2006-2019 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -7,10 +7,14 @@ """provides runtime services for templates, including Context, Namespace, and various helper functions.""" -from mako import exceptions, util, compat -from mako.compat import compat_builtins +import functools import sys +from mako import compat +from mako import exceptions +from mako import util +from mako.compat import compat_builtins + class Context(object): @@ -34,18 +38,19 @@ class Context(object): # "capture" function which proxies to the # generic "capture" function - self._data['capture'] = compat.partial(capture, self) + self._data["capture"] = functools.partial(capture, self) # "caller" stack used by def calls with content - self.caller_stack = self._data['caller'] = CallerStack() + self.caller_stack = self._data["caller"] = CallerStack() def _set_with_template(self, t): self._with_template = t illegal_names = t.reserved_names.intersection(self._data) if illegal_names: raise exceptions.NameConflictError( - "Reserved words passed to render(): %s" % - ", ".join(illegal_names)) + "Reserved words passed to render(): %s" + % ", ".join(illegal_names) + ) @property def lookup(self): @@ -177,14 +182,13 @@ class Context(object): c = self._copy() x = c._data - x.pop('self', None) - x.pop('parent', None) - x.pop('next', None) + x.pop("self", None) + x.pop("parent", None) + x.pop("next", None) return c class CallerStack(list): - def __init__(self): self.nextcaller = None @@ -231,6 +235,7 @@ class Undefined(object): def __bool__(self): return False + UNDEFINED = Undefined() STOP_RENDERING = "" @@ -342,7 +347,6 @@ class LoopContext(object): class _NSAttr(object): - def __init__(self, parent): self.__parent = parent @@ -373,9 +377,15 @@ class Namespace(object): """ - def __init__(self, name, context, - callables=None, inherits=None, - populate_self=True, calling_uri=None): + def __init__( + self, + name, + context, + callables=None, + inherits=None, + populate_self=True, + calling_uri=None, + ): self.name = name self.context = context self.inherits = inherits @@ -473,9 +483,12 @@ class Namespace(object): if key in self.context.namespaces: return self.context.namespaces[key] else: - ns = TemplateNamespace(uri, self.context._copy(), - templateuri=uri, - calling_uri=self._templateuri) + ns = TemplateNamespace( + uri, + self.context._copy(), + templateuri=uri, + calling_uri=self._templateuri, + ) self.context.namespaces[key] = ns return ns @@ -518,7 +531,7 @@ class Namespace(object): def _populate(self, d, l): for ident in l: - if ident == '*': + if ident == "*": for (k, v) in self._get_star(): d[k] = v else: @@ -536,8 +549,8 @@ class Namespace(object): val = getattr(self.inherits, key) else: raise AttributeError( - "Namespace '%s' has no member '%s'" % - (self.name, key)) + "Namespace '%s' has no member '%s'" % (self.name, key) + ) setattr(self, key, val) return val @@ -546,9 +559,17 @@ class TemplateNamespace(Namespace): """A :class:`.Namespace` specific to a :class:`.Template` instance.""" - def __init__(self, name, context, template=None, templateuri=None, - callables=None, inherits=None, - populate_self=True, calling_uri=None): + def __init__( + self, + name, + context, + template=None, + templateuri=None, + callables=None, + inherits=None, + populate_self=True, + calling_uri=None, + ): self.name = name self.context = context self.inherits = inherits @@ -556,8 +577,7 @@ class TemplateNamespace(Namespace): self.callables = dict([(c.__name__, c) for c in callables]) if templateuri is not None: - self.template = _lookup_template(context, templateuri, - calling_uri) + self.template = _lookup_template(context, templateuri, calling_uri) self._templateuri = self.template.module._template_uri elif template is not None: self.template = template @@ -566,9 +586,9 @@ class TemplateNamespace(Namespace): raise TypeError("'template' argument is required.") if populate_self: - lclcallable, lclcontext = \ - _populate_self_namespace(context, self.template, - self_ns=self) + lclcallable, lclcontext = _populate_self_namespace( + context, self.template, self_ns=self + ) @property def module(self): @@ -606,7 +626,8 @@ class TemplateNamespace(Namespace): def get(key): callable_ = self.template._get_def_callable(key) - return compat.partial(callable_, self.context) + return functools.partial(callable_, self.context) + for k in self.template.module._exports: yield (k, get(k)) @@ -615,14 +636,14 @@ class TemplateNamespace(Namespace): val = self.callables[key] elif self.template.has_def(key): callable_ = self.template._get_def_callable(key) - val = compat.partial(callable_, self.context) + val = functools.partial(callable_, self.context) elif self.inherits: val = getattr(self.inherits, key) else: raise AttributeError( - "Namespace '%s' has no member '%s'" % - (self.name, key)) + "Namespace '%s' has no member '%s'" % (self.name, key) + ) setattr(self, key, val) return val @@ -631,9 +652,16 @@ class ModuleNamespace(Namespace): """A :class:`.Namespace` specific to a Python module instance.""" - def __init__(self, name, context, module, - callables=None, inherits=None, - populate_self=True, calling_uri=None): + def __init__( + self, + name, + context, + module, + callables=None, + inherits=None, + populate_self=True, + calling_uri=None, + ): self.name = name self.context = context self.inherits = inherits @@ -641,7 +669,7 @@ class ModuleNamespace(Namespace): self.callables = dict([(c.__name__, c) for c in callables]) mod = __import__(module) - for token in module.split('.')[1:]: + for token in module.split(".")[1:]: mod = getattr(mod, token) self.module = mod @@ -657,23 +685,23 @@ class ModuleNamespace(Namespace): for key in self.callables: yield (key, self.callables[key]) for key in dir(self.module): - if key[0] != '_': + if key[0] != "_": callable_ = getattr(self.module, key) - if compat.callable(callable_): - yield key, compat.partial(callable_, self.context) + if callable(callable_): + yield key, functools.partial(callable_, self.context) def __getattr__(self, key): if key in self.callables: val = self.callables[key] elif hasattr(self.module, key): callable_ = getattr(self.module, key) - val = compat.partial(callable_, self.context) + val = functools.partial(callable_, self.context) elif self.inherits: val = getattr(self.inherits, key) else: raise AttributeError( - "Namespace '%s' has no member '%s'" % - (self.name, key)) + "Namespace '%s' has no member '%s'" % (self.name, key) + ) setattr(self, key, val) return val @@ -692,6 +720,7 @@ def supports_caller(func): return func(context, *args, **kwargs) finally: context.caller_stack._pop_frame() + return wrap_stackframe @@ -703,7 +732,7 @@ def capture(context, callable_, *args, **kwargs): """ - if not compat.callable(callable_): + if not callable(callable_): raise exceptions.RuntimeException( "capture() function expects a callable as " "its argument (i.e. capture(func, *args, **kwargs))" @@ -721,13 +750,16 @@ def _decorate_toplevel(fn): def go(context, *args, **kw): def y(*args, **kw): return render_fn(context, *args, **kw) + try: y.__name__ = render_fn.__name__[7:] except TypeError: # < Python 2.4 pass return fn(y)(context, *args, **kw) + return go + return decorate_render @@ -737,7 +769,9 @@ def _decorate_inline(context, fn): def go(*args, **kw): return dec(context, *args, **kw) + return go + return decorate_render @@ -747,8 +781,8 @@ def _include_file(context, uri, calling_uri, **kwargs): template = _lookup_template(context, uri, calling_uri) (callable_, ctx) = _populate_self_namespace( - context._clean_inheritance_tokens(), - template) + context._clean_inheritance_tokens(), template + ) kwargs = _kwargs_for_include(callable_, context._data, **kwargs) if template.include_error_handler: try: @@ -769,23 +803,25 @@ def _inherit_from(context, uri, calling_uri): if uri is None: return None template = _lookup_template(context, uri, calling_uri) - self_ns = context['self'] + self_ns = context["self"] ih = self_ns while ih.inherits is not None: ih = ih.inherits - lclcontext = context._locals({'next': ih}) - ih.inherits = TemplateNamespace("self:%s" % template.uri, - lclcontext, - template=template, - populate_self=False) - context._data['parent'] = lclcontext._data['local'] = ih.inherits - callable_ = getattr(template.module, '_mako_inherit', None) + lclcontext = context._locals({"next": ih}) + ih.inherits = TemplateNamespace( + "self:%s" % template.uri, + lclcontext, + template=template, + populate_self=False, + ) + context._data["parent"] = lclcontext._data["local"] = ih.inherits + callable_ = getattr(template.module, "_mako_inherit", None) if callable_ is not None: ret = callable_(template, lclcontext) if ret: return ret - gen_ns = getattr(template.module, '_mako_generate_namespaces', None) + gen_ns = getattr(template.module, "_mako_generate_namespaces", None) if gen_ns is not None: gen_ns(context) return (template.callable_, lclcontext) @@ -795,8 +831,9 @@ def _lookup_template(context, uri, relativeto): lookup = context._with_template.lookup if lookup is None: raise exceptions.TemplateLookupException( - "Template '%s' has no TemplateLookup associated" % - context._with_template.uri) + "Template '%s' has no TemplateLookup associated" + % context._with_template.uri + ) uri = lookup.adjust_uri(uri, relativeto) try: return lookup.get_template(uri) @@ -806,11 +843,14 @@ def _lookup_template(context, uri, relativeto): def _populate_self_namespace(context, template, self_ns=None): if self_ns is None: - self_ns = TemplateNamespace('self:%s' % template.uri, - context, template=template, - populate_self=False) - context._data['self'] = context._data['local'] = self_ns - if hasattr(template.module, '_mako_inherit'): + self_ns = TemplateNamespace( + "self:%s" % template.uri, + context, + template=template, + populate_self=False, + ) + context._data["self"] = context._data["local"] = self_ns + if hasattr(template.module, "_mako_inherit"): ret = template.module._mako_inherit(template, context) if ret: return ret @@ -829,18 +869,24 @@ def _render(template, callable_, args, data, as_unicode=False): buf = util.FastEncodingBuffer( as_unicode=as_unicode, encoding=template.output_encoding, - errors=template.encoding_errors) + errors=template.encoding_errors, + ) context = Context(buf, **data) context._outputting_as_unicode = as_unicode context._set_with_template(template) - _render_context(template, callable_, context, *args, - **_kwargs_for_callable(callable_, data)) + _render_context( + template, + callable_, + context, + *args, + **_kwargs_for_callable(callable_, data) + ) return context._pop_buffer().getvalue() def _kwargs_for_callable(callable_, data): - argspec = compat.inspect_func_args(callable_) + argspec = compat.inspect_getargspec(callable_) # for normal pages, **pageargs is usually present if argspec[2]: return data @@ -849,22 +895,23 @@ def _kwargs_for_callable(callable_, data): namedargs = argspec[0] + [v for v in argspec[1:3] if v is not None] kwargs = {} for arg in namedargs: - if arg != 'context' and arg in data and arg not in kwargs: + if arg != "context" and arg in data and arg not in kwargs: kwargs[arg] = data[arg] return kwargs def _kwargs_for_include(callable_, data, **kwargs): - argspec = compat.inspect_func_args(callable_) + argspec = compat.inspect_getargspec(callable_) namedargs = argspec[0] + [v for v in argspec[1:3] if v is not None] for arg in namedargs: - if arg != 'context' and arg in data and arg not in kwargs: + if arg != "context" and arg in data and arg not in kwargs: kwargs[arg] = data[arg] return kwargs def _render_context(tmpl, callable_, context, *args, **kwargs): import mako.template as template + # create polymorphic 'self' namespace for this # template with possibly updated context if not isinstance(tmpl, template.DefTemplate): @@ -886,8 +933,9 @@ def _exec_template(callable_, context, args=None, kwargs=None): be interpreted here. """ template = context._with_template - if template is not None and \ - (template.format_exceptions or template.error_handler): + if template is not None and ( + template.format_exceptions or template.error_handler + ): try: callable_(context, *args, **kwargs) except Exception: @@ -908,11 +956,15 @@ def _render_error(template, context, error): error_template = exceptions.html_error_template() if context._outputting_as_unicode: context._buffer_stack[:] = [ - util.FastEncodingBuffer(as_unicode=True)] + util.FastEncodingBuffer(as_unicode=True) + ] else: - context._buffer_stack[:] = [util.FastEncodingBuffer( - error_template.output_encoding, - error_template.encoding_errors)] + context._buffer_stack[:] = [ + util.FastEncodingBuffer( + error_template.output_encoding, + error_template.encoding_errors, + ) + ] context._set_with_template(error_template) error_template.render_context(context, error=error) diff --git a/server/www/packages/packages-linux/x64/mako/template.py b/server/www/packages/packages-linux/x64/mako/template.py index 329632c..937d15b 100644 --- a/server/www/packages/packages-linux/x64/mako/template.py +++ b/server/www/packages/packages-linux/x64/mako/template.py @@ -1,5 +1,5 @@ # mako/template.py -# Copyright (C) 2006-2016 the Mako authors and contributors +# Copyright 2006-2019 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php @@ -7,8 +7,7 @@ """Provides the Template class, a facade for parsing, generating and executing template strings, as well as template runtime operations.""" -from mako.lexer import Lexer -from mako import runtime, util, exceptions, codegen, cache, compat +import json import os import re import shutil @@ -18,6 +17,14 @@ import tempfile import types import weakref +from mako import cache +from mako import codegen +from mako import compat +from mako import exceptions +from mako import runtime +from mako import util +from mako.lexer import Lexer + class Template(object): @@ -230,41 +237,43 @@ class Template(object): lexer_cls = Lexer - def __init__(self, - text=None, - filename=None, - uri=None, - format_exceptions=False, - error_handler=None, - lookup=None, - output_encoding=None, - encoding_errors='strict', - module_directory=None, - cache_args=None, - cache_impl='beaker', - cache_enabled=True, - cache_type=None, - cache_dir=None, - cache_url=None, - module_filename=None, - input_encoding=None, - disable_unicode=False, - module_writer=None, - bytestring_passthrough=False, - default_filters=None, - buffer_filters=(), - strict_undefined=False, - imports=None, - future_imports=None, - enable_loop=True, - preprocessor=None, - lexer_cls=None, - include_error_handler=None): + def __init__( + self, + text=None, + filename=None, + uri=None, + format_exceptions=False, + error_handler=None, + lookup=None, + output_encoding=None, + encoding_errors="strict", + module_directory=None, + cache_args=None, + cache_impl="beaker", + cache_enabled=True, + cache_type=None, + cache_dir=None, + cache_url=None, + module_filename=None, + input_encoding=None, + disable_unicode=False, + module_writer=None, + bytestring_passthrough=False, + default_filters=None, + buffer_filters=(), + strict_undefined=False, + imports=None, + future_imports=None, + enable_loop=True, + preprocessor=None, + lexer_cls=None, + include_error_handler=None, + ): if uri: - self.module_id = re.sub(r'\W', "_", uri) + self.module_id = re.sub(r"\W", "_", uri) self.uri = uri elif filename: - self.module_id = re.sub(r'\W', "_", filename) + self.module_id = re.sub(r"\W", "_", filename) drive, path = os.path.splitdrive(filename) path = os.path.normpath(path).replace(os.path.sep, "/") self.uri = path @@ -278,9 +287,10 @@ class Template(object): u_norm = os.path.normpath(u_norm) if u_norm.startswith(".."): raise exceptions.TemplateLookupException( - "Template uri \"%s\" is invalid - " + 'Template uri "%s" is invalid - ' "it cannot be relative outside " - "of the root path." % self.uri) + "of the root path." % self.uri + ) self.input_encoding = input_encoding self.output_encoding = output_encoding @@ -293,17 +303,18 @@ class Template(object): if compat.py3k and disable_unicode: raise exceptions.UnsupportedError( - "Mako for Python 3 does not " - "support disabling Unicode") + "Mako for Python 3 does not " "support disabling Unicode" + ) elif output_encoding and disable_unicode: raise exceptions.UnsupportedError( "output_encoding must be set to " - "None when disable_unicode is used.") + "None when disable_unicode is used." + ) if default_filters is None: if compat.py3k or self.disable_unicode: - self.default_filters = ['str'] + self.default_filters = ["str"] else: - self.default_filters = ['unicode'] + self.default_filters = ["unicode"] else: self.default_filters = default_filters self.buffer_filters = buffer_filters @@ -320,7 +331,7 @@ class Template(object): (code, module) = _compile_text(self, text, filename) self._code = code self._source = text - ModuleInfo(module, None, self, filename, code, text) + ModuleInfo(module, None, self, filename, code, text, uri) elif filename is not None: # if template filename and a module directory, load # a filesystem-based module file, generating if needed @@ -329,8 +340,7 @@ class Template(object): elif module_directory is not None: path = os.path.abspath( os.path.join( - os.path.normpath(module_directory), - u_norm + ".py" + os.path.normpath(module_directory), u_norm + ".py" ) ) else: @@ -338,7 +348,8 @@ class Template(object): module = self._compile_from_file(path, filename) else: raise exceptions.RuntimeException( - "Template requires text or filename") + "Template requires text or filename" + ) self.module = module self.filename = filename @@ -351,8 +362,12 @@ class Template(object): self.module_directory = module_directory self._setup_cache_args( - cache_impl, cache_enabled, cache_args, - cache_type, cache_dir, cache_url + cache_impl, + cache_enabled, + cache_args, + cache_type, + cache_dir, + cache_url, ) @util.memoized_property @@ -360,11 +375,17 @@ class Template(object): if self.enable_loop: return codegen.RESERVED_NAMES else: - return codegen.RESERVED_NAMES.difference(['loop']) + return codegen.RESERVED_NAMES.difference(["loop"]) - def _setup_cache_args(self, - cache_impl, cache_enabled, cache_args, - cache_type, cache_dir, cache_url): + def _setup_cache_args( + self, + cache_impl, + cache_enabled, + cache_args, + cache_type, + cache_dir, + cache_url, + ): self.cache_impl = cache_impl self.cache_enabled = cache_enabled if cache_args: @@ -374,49 +395,42 @@ class Template(object): # transfer deprecated cache_* args if cache_type: - self.cache_args['type'] = cache_type + self.cache_args["type"] = cache_type if cache_dir: - self.cache_args['dir'] = cache_dir + self.cache_args["dir"] = cache_dir if cache_url: - self.cache_args['url'] = cache_url + self.cache_args["url"] = cache_url def _compile_from_file(self, path, filename): if path is not None: util.verify_directory(os.path.dirname(path)) filemtime = os.stat(filename)[stat.ST_MTIME] - if not os.path.exists(path) or \ - os.stat(path)[stat.ST_MTIME] < filemtime: + if ( + not os.path.exists(path) + or os.stat(path)[stat.ST_MTIME] < filemtime + ): data = util.read_file(filename) _compile_module_file( - self, - data, - filename, - path, - self.module_writer) + self, data, filename, path, self.module_writer + ) module = compat.load_module(self.module_id, path) del sys.modules[self.module_id] if module._magic_number != codegen.MAGIC_NUMBER: data = util.read_file(filename) _compile_module_file( - self, - data, - filename, - path, - self.module_writer) + self, data, filename, path, self.module_writer + ) module = compat.load_module(self.module_id, path) del sys.modules[self.module_id] - ModuleInfo(module, path, self, filename, None, None) + ModuleInfo(module, path, self, filename, None, None, None) else: # template filename and no module directory, compile code # in memory data = util.read_file(filename) - code, module = _compile_text( - self, - data, - filename) + code, module = _compile_text(self, data, filename) self._source = None self._code = code - ModuleInfo(module, None, self, filename, code, None) + ModuleInfo(module, None, self, filename, code, None, None) return module @property @@ -437,15 +451,15 @@ class Template(object): @property def cache_dir(self): - return self.cache_args['dir'] + return self.cache_args["dir"] @property def cache_url(self): - return self.cache_args['url'] + return self.cache_args["url"] @property def cache_type(self): - return self.cache_args['type'] + return self.cache_args["type"] def render(self, *args, **data): """Render the output of this template as a string. @@ -464,11 +478,9 @@ class Template(object): def render_unicode(self, *args, **data): """Render the output of this template as a unicode object.""" - return runtime._render(self, - self.callable_, - args, - data, - as_unicode=True) + return runtime._render( + self, self.callable_, args, data, as_unicode=True + ) def render_context(self, context, *args, **kwargs): """Render this :class:`.Template` with the given context. @@ -476,13 +488,9 @@ class Template(object): The data is written to the context's buffer. """ - if getattr(context, '_with_template', None) is None: + if getattr(context, "_with_template", None) is None: context._set_with_template(self) - runtime._render_context(self, - self.callable_, - context, - *args, - **kwargs) + runtime._render_context(self, self.callable_, context, *args, **kwargs) def has_def(self, name): return hasattr(self.module, "render_%s" % name) @@ -498,7 +506,7 @@ class Template(object): .. versionadded:: 1.0.4 """ - return [i[7:] for i in dir(self.module) if i[:7] == 'render_'] + return [i[7:] for i in dir(self.module) if i[:7] == "render_"] def _get_def_callable(self, name): return getattr(self.module, "render_%s" % name) @@ -526,28 +534,30 @@ class ModuleTemplate(Template): """ - def __init__(self, module, - module_filename=None, - template=None, - template_filename=None, - module_source=None, - template_source=None, - output_encoding=None, - encoding_errors='strict', - disable_unicode=False, - bytestring_passthrough=False, - format_exceptions=False, - error_handler=None, - lookup=None, - cache_args=None, - cache_impl='beaker', - cache_enabled=True, - cache_type=None, - cache_dir=None, - cache_url=None, - include_error_handler=None, - ): - self.module_id = re.sub(r'\W', "_", module._template_uri) + def __init__( + self, + module, + module_filename=None, + template=None, + template_filename=None, + module_source=None, + template_source=None, + output_encoding=None, + encoding_errors="strict", + disable_unicode=False, + bytestring_passthrough=False, + format_exceptions=False, + error_handler=None, + lookup=None, + cache_args=None, + cache_impl="beaker", + cache_enabled=True, + cache_type=None, + cache_dir=None, + cache_url=None, + include_error_handler=None, + ): + self.module_id = re.sub(r"\W", "_", module._template_uri) self.uri = module._template_uri self.input_encoding = module._source_encoding self.output_encoding = output_encoding @@ -558,21 +568,25 @@ class ModuleTemplate(Template): if compat.py3k and disable_unicode: raise exceptions.UnsupportedError( - "Mako for Python 3 does not " - "support disabling Unicode") + "Mako for Python 3 does not " "support disabling Unicode" + ) elif output_encoding and disable_unicode: raise exceptions.UnsupportedError( "output_encoding must be set to " - "None when disable_unicode is used.") + "None when disable_unicode is used." + ) self.module = module self.filename = template_filename - ModuleInfo(module, - module_filename, - self, - template_filename, - module_source, - template_source) + ModuleInfo( + module, + module_filename, + self, + template_filename, + module_source, + template_source, + module._template_uri, + ) self.callable_ = self.module.render_body self.format_exceptions = format_exceptions @@ -580,8 +594,12 @@ class ModuleTemplate(Template): self.include_error_handler = include_error_handler self.lookup = lookup self._setup_cache_args( - cache_impl, cache_enabled, cache_args, - cache_type, cache_dir, cache_url + cache_impl, + cache_enabled, + cache_args, + cache_type, + cache_dir, + cache_url, ) @@ -614,20 +632,25 @@ class ModuleInfo(object): source code based on a module's identifier. """ + _modules = weakref.WeakValueDictionary() - def __init__(self, - module, - module_filename, - template, - template_filename, - module_source, - template_source): + def __init__( + self, + module, + module_filename, + template, + template_filename, + module_source, + template_source, + template_uri, + ): self.module = module self.module_filename = module_filename self.template_filename = template_filename self.module_source = module_source self.template_source = template_source + self.template_uri = template_uri self._modules[module.__name__] = template._mmarker = self if module_filename: self._modules[module_filename] = self @@ -635,15 +658,15 @@ class ModuleInfo(object): @classmethod def get_module_source_metadata(cls, module_source, full_line_map=False): source_map = re.search( - r"__M_BEGIN_METADATA(.+?)__M_END_METADATA", - module_source, re.S).group(1) - source_map = compat.json.loads(source_map) - source_map['line_map'] = dict( - (int(k), int(v)) - for k, v in source_map['line_map'].items()) + r"__M_BEGIN_METADATA(.+?)__M_END_METADATA", module_source, re.S + ).group(1) + source_map = json.loads(source_map) + source_map["line_map"] = dict( + (int(k), int(v)) for k, v in source_map["line_map"].items() + ) if full_line_map: - f_line_map = source_map['full_line_map'] = [] - line_map = source_map['line_map'] + f_line_map = source_map["full_line_map"] = [] + line_map = source_map["line_map"] curr_templ_line = 1 for mod_line in range(1, max(line_map)): @@ -662,10 +685,12 @@ class ModuleInfo(object): @property def source(self): if self.template_source is not None: - if self.module._source_encoding and \ - not isinstance(self.template_source, compat.text_type): + if self.module._source_encoding and not isinstance( + self.template_source, compat.text_type + ): return self.template_source.decode( - self.module._source_encoding) + self.module._source_encoding + ) else: return self.template_source else: @@ -677,38 +702,46 @@ class ModuleInfo(object): def _compile(template, text, filename, generate_magic_comment): - lexer = template.lexer_cls(text, - filename, - disable_unicode=template.disable_unicode, - input_encoding=template.input_encoding, - preprocessor=template.preprocessor) + lexer = template.lexer_cls( + text, + filename, + disable_unicode=template.disable_unicode, + input_encoding=template.input_encoding, + preprocessor=template.preprocessor, + ) node = lexer.parse() - source = codegen.compile(node, - template.uri, - filename, - default_filters=template.default_filters, - buffer_filters=template.buffer_filters, - imports=template.imports, - future_imports=template.future_imports, - source_encoding=lexer.encoding, - generate_magic_comment=generate_magic_comment, - disable_unicode=template.disable_unicode, - strict_undefined=template.strict_undefined, - enable_loop=template.enable_loop, - reserved_names=template.reserved_names) + source = codegen.compile( + node, + template.uri, + filename, + default_filters=template.default_filters, + buffer_filters=template.buffer_filters, + imports=template.imports, + future_imports=template.future_imports, + source_encoding=lexer.encoding, + generate_magic_comment=generate_magic_comment, + disable_unicode=template.disable_unicode, + strict_undefined=template.strict_undefined, + enable_loop=template.enable_loop, + reserved_names=template.reserved_names, + ) return source, lexer def _compile_text(template, text, filename): identifier = template.module_id - source, lexer = _compile(template, text, filename, - generate_magic_comment=template.disable_unicode) + source, lexer = _compile( + template, + text, + filename, + generate_magic_comment=template.disable_unicode, + ) cid = identifier if not compat.py3k and isinstance(cid, compat.text_type): cid = cid.encode() module = types.ModuleType(cid) - code = compile(source, cid, 'exec') + code = compile(source, cid, "exec") # this exec() works for 2.4->3.3. exec(code, module.__dict__, module.__dict__) @@ -716,11 +749,12 @@ def _compile_text(template, text, filename): def _compile_module_file(template, text, filename, outputpath, module_writer): - source, lexer = _compile(template, text, filename, - generate_magic_comment=True) + source, lexer = _compile( + template, text, filename, generate_magic_comment=True + ) if isinstance(source, compat.text_type): - source = source.encode(lexer.encoding or 'ascii') + source = source.encode(lexer.encoding or "ascii") if module_writer: module_writer(source, outputpath) @@ -737,9 +771,9 @@ def _compile_module_file(template, text, filename, outputpath, module_writer): def _get_module_info_from_callable(callable_): if compat.py3k: - return _get_module_info(callable_.__globals__['__name__']) + return _get_module_info(callable_.__globals__["__name__"]) else: - return _get_module_info(callable_.func_globals['__name__']) + return _get_module_info(callable_.func_globals["__name__"]) def _get_module_info(filename): diff --git a/server/www/packages/packages-linux/x64/mako/util.py b/server/www/packages/packages-linux/x64/mako/util.py index 2f089ff..07f7531 100644 --- a/server/www/packages/packages-linux/x64/mako/util.py +++ b/server/www/packages/packages-linux/x64/mako/util.py @@ -1,15 +1,17 @@ # mako/util.py -# Copyright (C) 2006-2016 the Mako authors and contributors +# Copyright 2006-2019 the Mako authors and contributors # # This module is part of Mako and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -import re -import collections import codecs -import os -from mako import compat +import collections import operator +import os +import re +import timeit + +from mako import compat def update_wrapper(decorated, fn): @@ -19,7 +21,6 @@ def update_wrapper(decorated, fn): class PluginLoader(object): - def __init__(self, group): self.group = group self.impls = {} @@ -29,16 +30,16 @@ class PluginLoader(object): return self.impls[name]() else: import pkg_resources - for impl in pkg_resources.iter_entry_points( - self.group, - name): + + for impl in pkg_resources.iter_entry_points(self.group, name): self.impls[name] = impl.load return impl.load() else: from mako import exceptions + raise exceptions.RuntimeException( - "Can't load plugin %s %s" % - (self.group, name)) + "Can't load plugin %s %s" % (self.group, name) + ) def register(self, name, modulepath, objname): def load(): @@ -46,18 +47,19 @@ class PluginLoader(object): for token in modulepath.split(".")[1:]: mod = getattr(mod, token) return getattr(mod, objname) + self.impls[name] = load -def verify_directory(dir): +def verify_directory(dir_): """create and/or verify a filesystem directory.""" tries = 0 - while not os.path.exists(dir): + while not os.path.exists(dir_): try: tries += 1 - os.makedirs(dir, compat.octal("0775")) + os.makedirs(dir_, compat.octal("0775")) except: if tries > 5: raise @@ -109,11 +111,15 @@ class memoized_instancemethod(object): def oneshot(*args, **kw): result = self.fget(obj, *args, **kw) - memo = lambda *a, **kw: result + + def memo(*a, **kw): + return result + memo.__name__ = self.__name__ memo.__doc__ = self.__doc__ obj.__dict__[self.__name__] = memo return result + oneshot.__name__ = self.__name__ oneshot.__doc__ = self.__doc__ return oneshot @@ -137,13 +143,13 @@ class FastEncodingBuffer(object): """a very rudimentary buffer that is faster than StringIO, but doesn't crash on unicode data like cStringIO.""" - def __init__(self, encoding=None, errors='strict', as_unicode=False): + def __init__(self, encoding=None, errors="strict", as_unicode=False): self.data = collections.deque() self.encoding = encoding if as_unicode: - self.delim = compat.u('') + self.delim = compat.u("") else: - self.delim = '' + self.delim = "" self.as_unicode = as_unicode self.errors = errors self.write = self.data.append @@ -154,8 +160,9 @@ class FastEncodingBuffer(object): def getvalue(self): if self.encoding: - return self.delim.join(self.data).encode(self.encoding, - self.errors) + return self.delim.join(self.data).encode( + self.encoding, self.errors + ) else: return self.delim.join(self.data) @@ -171,22 +178,21 @@ class LRUCache(dict): """ class _Item(object): - def __init__(self, key, value): self.key = key self.value = value - self.timestamp = compat.time_func() + self.timestamp = timeit.default_timer() def __repr__(self): return repr(self.value) - def __init__(self, capacity, threshold=.5): + def __init__(self, capacity, threshold=0.5): self.capacity = capacity self.threshold = threshold def __getitem__(self, key): item = dict.__getitem__(self, key) - item.timestamp = compat.time_func() + item.timestamp = timeit.default_timer() return item.value def values(self): @@ -210,9 +216,12 @@ class LRUCache(dict): def _manage_size(self): while len(self) > self.capacity + self.capacity * self.threshold: - bytime = sorted(dict.values(self), - key=operator.attrgetter('timestamp'), reverse=True) - for item in bytime[self.capacity:]: + bytime = sorted( + dict.values(self), + key=operator.attrgetter("timestamp"), + reverse=True, + ) + for item in bytime[self.capacity :]: try: del self[item.key] except KeyError: @@ -220,10 +229,11 @@ class LRUCache(dict): # broke in on us. loop around and try again break + # Regexp to match python magic encoding line _PYTHON_MAGIC_COMMENT_re = re.compile( - r'[ \t\f]* \# .* coding[=:][ \t]*([-\w.]+)', - re.VERBOSE) + r"[ \t\f]* \# .* coding[=:][ \t]*([-\w.]+)", re.VERBOSE +) def parse_encoding(fp): @@ -242,13 +252,14 @@ def parse_encoding(fp): line1 = fp.readline() has_bom = line1.startswith(codecs.BOM_UTF8) if has_bom: - line1 = line1[len(codecs.BOM_UTF8):] + line1 = line1[len(codecs.BOM_UTF8) :] - m = _PYTHON_MAGIC_COMMENT_re.match(line1.decode('ascii', 'ignore')) + m = _PYTHON_MAGIC_COMMENT_re.match(line1.decode("ascii", "ignore")) if not m: try: import parser - parser.suite(line1.decode('ascii', 'ignore')) + + parser.suite(line1.decode("ascii", "ignore")) except (ImportError, SyntaxError): # Either it's a real syntax error, in which case the source # is not valid python source, or line2 is a continuation of @@ -258,14 +269,16 @@ def parse_encoding(fp): else: line2 = fp.readline() m = _PYTHON_MAGIC_COMMENT_re.match( - line2.decode('ascii', 'ignore')) + line2.decode("ascii", "ignore") + ) if has_bom: if m: raise SyntaxError( "python refuses to compile code with both a UTF8" - " byte-order-mark and a magic encoding comment") - return 'utf_8' + " byte-order-mark and a magic encoding comment" + ) + return "utf_8" elif m: return m.group(1) else: @@ -289,10 +302,11 @@ def restore__ast(_ast): """Attempt to restore the required classes to the _ast module if it appears to be missing them """ - if hasattr(_ast, 'AST'): + if hasattr(_ast, "AST"): return _ast.PyCF_ONLY_AST = 2 << 9 - m = compile("""\ + m = compile( + """\ def foo(): pass class Bar(object): pass if False: pass @@ -305,13 +319,17 @@ baz = 'mako' baz and 'foo' or 'bar' (mako is baz == baz) is not baz != mako mako > baz < mako >= baz <= mako -mako in baz not in mako""", '', 'exec', _ast.PyCF_ONLY_AST) +mako in baz not in mako""", + "", + "exec", + _ast.PyCF_ONLY_AST, + ) _ast.Module = type(m) for cls in _ast.Module.__mro__: - if cls.__name__ == 'mod': + if cls.__name__ == "mod": _ast.mod = cls - elif cls.__name__ == 'AST': + elif cls.__name__ == "AST": _ast.AST = cls _ast.FunctionDef = type(m.body[0]) @@ -361,7 +379,7 @@ mako in baz not in mako""", '', 'exec', _ast.PyCF_ONLY_AST) _ast.NotIn = type(m.body[12].value.ops[1]) -def read_file(path, mode='rb'): +def read_file(path, mode="rb"): fp = open(path, mode) try: data = fp.read() diff --git a/server/www/packages/packages-linux/x64/psutil/__init__.py b/server/www/packages/packages-linux/x64/psutil/__init__.py index c2a83fb..3f38058 100644 --- a/server/www/packages/packages-linux/x64/psutil/__init__.py +++ b/server/www/packages/packages-linux/x64/psutil/__init__.py @@ -17,7 +17,7 @@ sensors) in Python. Supported platforms: - Sun Solaris - AIX -Works with Python versions from 2.6 to 3.X. +Works with Python versions from 2.6 to 3.4+. """ from __future__ import division @@ -25,12 +25,12 @@ from __future__ import division import collections import contextlib import datetime -import errno import functools import os import signal import subprocess import sys +import threading import time try: import pwd @@ -43,6 +43,8 @@ from ._common import memoize from ._common import memoize_when_activated from ._common import wrap_numbers as _wrap_numbers from ._compat import long +from ._compat import PermissionError +from ._compat import ProcessLookupError from ._compat import PY3 as _PY3 from ._common import STATUS_DEAD @@ -86,12 +88,6 @@ from ._common import POSIX # NOQA from ._common import SUNOS from ._common import WINDOWS -from ._exceptions import AccessDenied -from ._exceptions import Error -from ._exceptions import NoSuchProcess -from ._exceptions import TimeoutExpired -from ._exceptions import ZombieProcess - if LINUX: # This is public API and it will be retrieved from _pslinux.py # via sys.modules. @@ -151,6 +147,10 @@ elif WINDOWS: from ._psutil_windows import NORMAL_PRIORITY_CLASS # NOQA from ._psutil_windows import REALTIME_PRIORITY_CLASS # NOQA from ._pswindows import CONN_DELETE_TCB # NOQA + from ._pswindows import IOPRIO_VERYLOW # NOQA + from ._pswindows import IOPRIO_LOW # NOQA + from ._pswindows import IOPRIO_NORMAL # NOQA + from ._pswindows import IOPRIO_HIGH # NOQA elif MACOS: from . import _psosx as _psplatform @@ -211,23 +211,26 @@ __all__ = [ "pid_exists", "pids", "process_iter", "wait_procs", # proc "virtual_memory", "swap_memory", # memory "cpu_times", "cpu_percent", "cpu_times_percent", "cpu_count", # cpu - "cpu_stats", # "cpu_freq", + "cpu_stats", # "cpu_freq", "getloadavg" "net_io_counters", "net_connections", "net_if_addrs", # network "net_if_stats", "disk_io_counters", "disk_partitions", "disk_usage", # disk # "sensors_temperatures", "sensors_battery", "sensors_fans" # sensors "users", "boot_time", # others ] + + __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" -__version__ = "5.4.8" +__version__ = "5.6.5" version_info = tuple([int(num) for num in __version__.split('.')]) + +_timer = getattr(time, 'monotonic', time.time) AF_LINK = _psplatform.AF_LINK POWER_TIME_UNLIMITED = _common.POWER_TIME_UNLIMITED POWER_TIME_UNKNOWN = _common.POWER_TIME_UNKNOWN _TOTAL_PHYMEM = None -_timer = getattr(time, 'monotonic', time.time) - +_LOWEST_PID = None # Sanity check in case the user messed up with psutil installation # or did something weird with sys.path. In this case we might end @@ -251,6 +254,112 @@ if (int(__version__.replace('.', '')) != raise ImportError(msg) +# ===================================================================== +# --- Exceptions +# ===================================================================== + + +class Error(Exception): + """Base exception class. All other psutil exceptions inherit + from this one. + """ + + def __init__(self, msg=""): + Exception.__init__(self, msg) + self.msg = msg + + def __repr__(self): + ret = "psutil.%s %s" % (self.__class__.__name__, self.msg) + return ret.strip() + + __str__ = __repr__ + + +class NoSuchProcess(Error): + """Exception raised when a process with a certain PID doesn't + or no longer exists. + """ + + def __init__(self, pid, name=None, msg=None): + Error.__init__(self, msg) + self.pid = pid + self.name = name + self.msg = msg + if msg is None: + if name: + details = "(pid=%s, name=%s)" % (self.pid, repr(self.name)) + else: + details = "(pid=%s)" % self.pid + self.msg = "process no longer exists " + details + + +class ZombieProcess(NoSuchProcess): + """Exception raised when querying a zombie process. This is + raised on macOS, BSD and Solaris only, and not always: depending + on the query the OS may be able to succeed anyway. + On Linux all zombie processes are querable (hence this is never + raised). Windows doesn't have zombie processes. + """ + + def __init__(self, pid, name=None, ppid=None, msg=None): + NoSuchProcess.__init__(self, msg) + self.pid = pid + self.ppid = ppid + self.name = name + self.msg = msg + if msg is None: + args = ["pid=%s" % pid] + if name: + args.append("name=%s" % repr(self.name)) + if ppid: + args.append("ppid=%s" % self.ppid) + details = "(%s)" % ", ".join(args) + self.msg = "process still exists but it's a zombie " + details + + +class AccessDenied(Error): + """Exception raised when permission to perform an action is denied.""" + + def __init__(self, pid=None, name=None, msg=None): + Error.__init__(self, msg) + self.pid = pid + self.name = name + self.msg = msg + if msg is None: + if (pid is not None) and (name is not None): + self.msg = "(pid=%s, name=%s)" % (pid, repr(name)) + elif (pid is not None): + self.msg = "(pid=%s)" % self.pid + else: + self.msg = "" + + +class TimeoutExpired(Error): + """Raised on Process.wait(timeout) if timeout expires and process + is still alive. + """ + + def __init__(self, seconds, pid=None, name=None): + Error.__init__(self, "timeout after %s seconds" % seconds) + self.seconds = seconds + self.pid = pid + self.name = name + if (pid is not None) and (name is not None): + self.msg += " (pid=%s, name=%s)" % (pid, repr(name)) + elif (pid is not None): + self.msg += " (pid=%s)" % self.pid + + +# Push exception classes into platform specific module namespace. +_psplatform.NoSuchProcess = NoSuchProcess +_psplatform.ZombieProcess = ZombieProcess +_psplatform.AccessDenied = AccessDenied +_psplatform.TimeoutExpired = TimeoutExpired +if POSIX: + from . import _psposix + _psposix.TimeoutExpired = TimeoutExpired + + # ===================================================================== # --- Utils # ===================================================================== @@ -352,7 +461,7 @@ class Process(object): self._create_time = None self._gone = False self._hash = None - self._oneshot_inctx = False + self._lock = threading.RLock() # used for caching on Windows only (on POSIX ppid may change) self._ppid = None # platform-specific modules define an _psplatform.Process @@ -456,40 +565,45 @@ class Process(object): ... >>> """ - if self._oneshot_inctx: - # NOOP: this covers the use case where the user enters the - # context twice. Since as_dict() internally uses oneshot() - # I expect that the code below will be a pretty common - # "mistake" that the user will make, so let's guard - # against that: - # - # >>> with p.oneshot(): - # ... p.as_dict() - # ... - yield - else: - self._oneshot_inctx = True - try: - # cached in case cpu_percent() is used - self.cpu_times.cache_activate() - # cached in case memory_percent() is used - self.memory_info.cache_activate() - # cached in case parent() is used - self.ppid.cache_activate() - # cached in case username() is used - if POSIX: - self.uids.cache_activate() - # specific implementation cache - self._proc.oneshot_enter() + with self._lock: + if hasattr(self, "_cache"): + # NOOP: this covers the use case where the user enters the + # context twice: + # + # >>> with p.oneshot(): + # ... with p.oneshot(): + # ... + # + # Also, since as_dict() internally uses oneshot() + # I expect that the code below will be a pretty common + # "mistake" that the user will make, so let's guard + # against that: + # + # >>> with p.oneshot(): + # ... p.as_dict() + # ... yield - finally: - self.cpu_times.cache_deactivate() - self.memory_info.cache_deactivate() - self.ppid.cache_deactivate() - if POSIX: - self.uids.cache_deactivate() - self._proc.oneshot_exit() - self._oneshot_inctx = False + else: + try: + # cached in case cpu_percent() is used + self.cpu_times.cache_activate(self) + # cached in case memory_percent() is used + self.memory_info.cache_activate(self) + # cached in case parent() is used + self.ppid.cache_activate(self) + # cached in case username() is used + if POSIX: + self.uids.cache_activate(self) + # specific implementation cache + self._proc.oneshot_enter() + yield + finally: + self.cpu_times.cache_deactivate(self) + self.memory_info.cache_deactivate(self) + self.ppid.cache_deactivate(self) + if POSIX: + self.uids.cache_deactivate(self) + self._proc.oneshot_exit() def as_dict(self, attrs=None, ad_value=None): """Utility method returning process information as a @@ -540,6 +654,9 @@ class Process(object): checking whether PID has been reused. If no parent is known return None. """ + lowest_pid = _LOWEST_PID if _LOWEST_PID is not None else pids()[0] + if self.pid == lowest_pid: + return None ppid = self.ppid() if ppid is not None: ctime = self.create_time() @@ -551,6 +668,17 @@ class Process(object): except NoSuchProcess: pass + def parents(self): + """Return the parents of this process as a list of Process + instances. If no parents are known return an empty list. + """ + parents = [] + proc = self.parent() + while proc is not None: + parents.append(proc) + proc = proc.parent() + return parents + def is_running(self): """Return whether this process is running. It also checks if PID has been reused by another process in @@ -799,9 +927,6 @@ class Process(object): (and set). (Windows, Linux and BSD only). """ - # Automatically remove duplicates both on get and - # set (for get it's not really necessary, it's - # just for extra safety). if cpus is None: return list(set(self._proc.cpu_affinity_get())) else: @@ -825,7 +950,7 @@ class Process(object): """ return self._proc.cpu_num() - # Linux, macOS and Windows only + # Linux, macOS, Windows, Solaris, AIX if hasattr(_psplatform.Process, "environ"): def environ(self): @@ -1095,7 +1220,6 @@ class Process(object): return (value / float(total_phymem)) * 100 if hasattr(_psplatform.Process, "memory_maps"): - # Available everywhere except OpenBSD and NetBSD. def memory_maps(self, grouped=True): """Return process' mapped memory regions as a list of namedtuples whose fields are variable depending on the platform. @@ -1167,18 +1291,16 @@ class Process(object): "calling process (os.getpid()) instead of PID 0") try: os.kill(self.pid, sig) - except OSError as err: - if err.errno == errno.ESRCH: - if OPENBSD and pid_exists(self.pid): - # We do this because os.kill() lies in case of - # zombie processes. - raise ZombieProcess(self.pid, self._name, self._ppid) - else: - self._gone = True - raise NoSuchProcess(self.pid, self._name) - if err.errno in (errno.EPERM, errno.EACCES): - raise AccessDenied(self.pid, self._name) - raise + except ProcessLookupError: + if OPENBSD and pid_exists(self.pid): + # We do this because os.kill() lies in case of + # zombie processes. + raise ZombieProcess(self.pid, self._name, self._ppid) + else: + self._gone = True + raise NoSuchProcess(self.pid, self._name) + except PermissionError: + raise AccessDenied(self.pid, self._name) @_assert_pid_not_reused def send_signal(self, sig): @@ -1298,7 +1420,7 @@ class Popen(Process): http://bugs.python.org/issue6973. For a complete documentation refer to: - http://docs.python.org/library/subprocess.html + http://docs.python.org/3/library/subprocess.html """ def __init__(self, *args, **kwargs): @@ -1354,7 +1476,7 @@ class Popen(Process): _as_dict_attrnames = set( [x for x in dir(Process) if not x.startswith('_') and x not in ['send_signal', 'suspend', 'resume', 'terminate', 'kill', 'wait', - 'is_running', 'as_dict', 'parent', 'children', 'rlimit', + 'is_running', 'as_dict', 'parent', 'parents', 'children', 'rlimit', 'memory_info_ex', 'oneshot']]) @@ -1365,7 +1487,10 @@ _as_dict_attrnames = set( def pids(): """Return a list of current running PIDs.""" - return _psplatform.pids() + global _LOWEST_PID + ret = sorted(_psplatform.pids()) + _LOWEST_PID = ret[0] + return ret def pid_exists(pid): @@ -1387,6 +1512,7 @@ def pid_exists(pid): _pmap = {} +_lock = threading.Lock() def process_iter(attrs=None, ad_value=None): @@ -1414,21 +1540,26 @@ def process_iter(attrs=None, ad_value=None): proc = Process(pid) if attrs is not None: proc.info = proc.as_dict(attrs=attrs, ad_value=ad_value) - _pmap[proc.pid] = proc + with _lock: + _pmap[proc.pid] = proc return proc def remove(pid): - _pmap.pop(pid, None) + with _lock: + _pmap.pop(pid, None) a = set(pids()) b = set(_pmap.keys()) new_pids = a - b gone_pids = b - a - for pid in gone_pids: remove(pid) - for pid, proc in sorted(list(_pmap.items()) + - list(dict.fromkeys(new_pids).items())): + + with _lock: + ls = sorted(list(_pmap.items()) + + list(dict.fromkeys(new_pids).items())) + + for pid, proc in ls: try: if proc is None: # new process yield add(pid) @@ -1885,6 +2016,17 @@ if hasattr(_psplatform, "cpu_freq"): __all__.append("cpu_freq") +if hasattr(os, "getloadavg") or hasattr(_psplatform, "getloadavg"): + # Perform this hasattr check once on import time to either use the + # platform based code or proxy straight from the os module. + if hasattr(os, "getloadavg"): + getloadavg = os.getloadavg + else: + getloadavg = _psplatform.getloadavg + + __all__.append("getloadavg") + + # ===================================================================== # --- system memory related functions # ===================================================================== @@ -1910,7 +2052,7 @@ def virtual_memory(): - used: memory used, calculated differently depending on the platform and designed for informational purposes only: - macOS: active + inactive + wired + macOS: active + wired BSD: active + wired + cached Linux: total - free @@ -2306,19 +2448,16 @@ if WINDOWS: def test(): # pragma: no cover - """List info of all currently running processes emulating ps aux - output. - """ + from ._common import bytes2human + from ._compat import get_terminal_size + today_day = datetime.date.today() - templ = "%-10s %5s %4s %7s %7s %-13s %5s %7s %s" - attrs = ['pid', 'memory_percent', 'name', 'cpu_times', 'create_time', - 'memory_info'] - if POSIX: - attrs.append('uids') - attrs.append('terminal') - print(templ % ("USER", "PID", "%MEM", "VSZ", "RSS", "TTY", "START", "TIME", - "COMMAND")) - for p in process_iter(attrs=attrs, ad_value=''): + templ = "%-10s %5s %5s %7s %7s %5s %6s %6s %6s %s" + attrs = ['pid', 'memory_percent', 'name', 'cmdline', 'cpu_times', + 'create_time', 'memory_info', 'status', 'nice', 'username'] + print(templ % ("USER", "PID", "%MEM", "VSZ", "RSS", "NICE", + "STATUS", "START", "TIME", "CMDLINE")) + for p in process_iter(attrs, ad_value=None): if p.info['create_time']: ctime = datetime.datetime.fromtimestamp(p.info['create_time']) if ctime.date() == today_day: @@ -2327,30 +2466,46 @@ def test(): # pragma: no cover ctime = ctime.strftime("%b%d") else: ctime = '' - cputime = time.strftime("%M:%S", - time.localtime(sum(p.info['cpu_times']))) - try: - user = p.username() - except Error: - user = '' - if WINDOWS and '\\' in user: + if p.info['cpu_times']: + cputime = time.strftime("%M:%S", + time.localtime(sum(p.info['cpu_times']))) + else: + cputime = '' + + user = p.info['username'] or '' + if not user and POSIX: + try: + user = p.uids()[0] + except Error: + pass + if user and WINDOWS and '\\' in user: user = user.split('\\')[1] - vms = p.info['memory_info'] and \ - int(p.info['memory_info'].vms / 1024) or '?' - rss = p.info['memory_info'] and \ - int(p.info['memory_info'].rss / 1024) or '?' - memp = p.info['memory_percent'] and \ - round(p.info['memory_percent'], 1) or '?' - print(templ % ( + user = user[:9] + vms = bytes2human(p.info['memory_info'].vms) if \ + p.info['memory_info'] is not None else '' + rss = bytes2human(p.info['memory_info'].rss) if \ + p.info['memory_info'] is not None else '' + memp = round(p.info['memory_percent'], 1) if \ + p.info['memory_percent'] is not None else '' + nice = int(p.info['nice']) if p.info['nice'] else '' + if p.info['cmdline']: + cmdline = ' '.join(p.info['cmdline']) + else: + cmdline = p.info['name'] + status = p.info['status'][:5] if p.info['status'] else '' + + line = templ % ( user[:10], p.info['pid'], memp, vms, rss, - p.info.get('terminal', '') or '?', + nice, + status, ctime, cputime, - p.info['name'].strip() or '?')) + cmdline) + print(line[:get_terminal_size()[0]]) del memoize, memoize_when_activated, division, deprecated_method diff --git a/server/www/packages/packages-linux/x64/psutil/_common.py b/server/www/packages/packages-linux/x64/psutil/_common.py index bee9579..126d9d6 100644 --- a/server/www/packages/packages-linux/x64/psutil/_common.py +++ b/server/www/packages/packages-linux/x64/psutil/_common.py @@ -64,6 +64,7 @@ __all__ = [ 'conn_tmap', 'deprecated_method', 'isfile_strict', 'memoize', 'parse_environ_block', 'path_exists_strict', 'usage_percent', 'supports_ipv6', 'sockfam_to_enum', 'socktype_to_enum', "wrap_numbers", + 'bytes2human', 'conn_to_ntuple', ] @@ -256,8 +257,6 @@ if AF_UNIX is not None: "unix": ([AF_UNIX], [SOCK_STREAM, SOCK_DGRAM]), }) -del AF_INET, AF_UNIX, SOCK_STREAM, SOCK_DGRAM - # =================================================================== # --- utils @@ -267,12 +266,12 @@ del AF_INET, AF_UNIX, SOCK_STREAM, SOCK_DGRAM def usage_percent(used, total, round_=None): """Calculate percentage usage of 'used' against 'total'.""" try: - ret = (used / total) * 100 + ret = (float(used) / total) * 100 except ZeroDivisionError: - ret = 0.0 if isinstance(used, float) or isinstance(total, float) else 0 - if round_ is not None: - return round(ret, round_) + return 0.0 else: + if round_ is not None: + ret = round(ret, round_) return ret @@ -327,7 +326,7 @@ def memoize_when_activated(fun): 1 >>> >>> # activated - >>> foo.cache_activate() + >>> foo.cache_activate(self) >>> foo() 1 >>> foo() @@ -336,26 +335,30 @@ def memoize_when_activated(fun): """ @functools.wraps(fun) def wrapper(self): - if not wrapper.cache_activated: + try: + # case 1: we previously entered oneshot() ctx + ret = self._cache[fun] + except AttributeError: + # case 2: we never entered oneshot() ctx return fun(self) - else: - try: - ret = cache[fun] - except KeyError: - ret = cache[fun] = fun(self) - return ret + except KeyError: + # case 3: we entered oneshot() ctx but there's no cache + # for this entry yet + ret = self._cache[fun] = fun(self) + return ret - def cache_activate(): - """Activate cache.""" - wrapper.cache_activated = True + def cache_activate(proc): + """Activate cache. Expects a Process instance. Cache will be + stored as a "_cache" instance attribute.""" + proc._cache = {} - def cache_deactivate(): + def cache_deactivate(proc): """Deactivate and clear cache.""" - wrapper.cache_activated = False - cache.clear() + try: + del proc._cache + except AttributeError: + pass - cache = {} - wrapper.cache_activated = False wrapper.cache_activate = cache_activate wrapper.cache_deactivate = cache_deactivate return wrapper @@ -442,7 +445,7 @@ def sockfam_to_enum(num): else: # pragma: no cover try: return socket.AddressFamily(num) - except (ValueError, AttributeError): + except ValueError: return num @@ -454,11 +457,30 @@ def socktype_to_enum(num): return num else: # pragma: no cover try: - return socket.AddressType(num) - except (ValueError, AttributeError): + return socket.SocketKind(num) + except ValueError: return num +def conn_to_ntuple(fd, fam, type_, laddr, raddr, status, status_map, pid=None): + """Convert a raw connection tuple to a proper ntuple.""" + if fam in (socket.AF_INET, AF_INET6): + if laddr: + laddr = addr(*laddr) + if raddr: + raddr = addr(*raddr) + if type_ == socket.SOCK_STREAM and fam in (AF_INET, AF_INET6): + status = status_map.get(status, CONN_NONE) + else: + status = CONN_NONE # ignore whatever C returned to us + fam = sockfam_to_enum(fam) + type_ = socktype_to_enum(type_) + if pid is None: + return pconn(fd, fam, type_, laddr, raddr, status) + else: + return sconn(fd, fam, type_, laddr, raddr, status, pid) + + def deprecated_method(replacement): """A decorator which can be used to mark a method as deprecated 'replcement' is the method name which will be called instead. @@ -471,7 +493,7 @@ def deprecated_method(replacement): @functools.wraps(fun) def inner(self, *args, **kwargs): - warnings.warn(msg, category=FutureWarning, stacklevel=2) + warnings.warn(msg, category=DeprecationWarning, stacklevel=2) return getattr(self, replacement)(*args, **kwargs) return inner return outer @@ -594,3 +616,36 @@ def open_text(fname, **kwargs): kwargs.setdefault('encoding', ENCODING) kwargs.setdefault('errors', ENCODING_ERRS) return open(fname, "rt", **kwargs) + + +def bytes2human(n, format="%(value).1f%(symbol)s"): + """Used by various scripts. See: + http://goo.gl/zeJZl + + >>> bytes2human(10000) + '9.8K' + >>> bytes2human(100001221) + '95.4M' + """ + symbols = ('B', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y') + prefix = {} + for i, s in enumerate(symbols[1:]): + prefix[s] = 1 << (i + 1) * 10 + for symbol in reversed(symbols[1:]): + if n >= prefix[symbol]: + value = float(n) / prefix[symbol] + return format % locals() + return format % dict(symbol=symbols[0], value=n) + + +def get_procfs_path(): + """Return updated psutil.PROCFS_PATH constant.""" + return sys.modules['psutil'].PROCFS_PATH + + +if PY3: + def decode(s): + return s.decode(encoding=ENCODING, errors=ENCODING_ERRS) +else: + def decode(s): + return s diff --git a/server/www/packages/packages-linux/x64/psutil/_compat.py b/server/www/packages/packages-linux/x64/psutil/_compat.py index 08aefe4..07ab909 100644 --- a/server/www/packages/packages-linux/x64/psutil/_compat.py +++ b/server/www/packages/packages-linux/x64/psutil/_compat.py @@ -5,12 +5,15 @@ """Module which provides compatibility with older Python versions.""" import collections +import errno import functools import os import sys __all__ = ["PY3", "long", "xrange", "unicode", "basestring", "u", "b", - "lru_cache", "which"] + "lru_cache", "which", "get_terminal_size", + "FileNotFoundError", "PermissionError", "ProcessLookupError", + "InterruptedError", "ChildProcessError", "FileExistsError"] PY3 = sys.version_info[0] == 3 @@ -38,6 +41,73 @@ else: return s +# --- exceptions + + +if PY3: + FileNotFoundError = FileNotFoundError # NOQA + PermissionError = PermissionError # NOQA + ProcessLookupError = ProcessLookupError # NOQA + InterruptedError = InterruptedError # NOQA + ChildProcessError = ChildProcessError # NOQA + FileExistsError = FileExistsError # NOQA +else: + # https://github.com/PythonCharmers/python-future/blob/exceptions/ + # src/future/types/exceptions/pep3151.py + + def instance_checking_exception(base_exception=Exception): + def wrapped(instance_checker): + class TemporaryClass(base_exception): + + def __init__(self, *args, **kwargs): + if len(args) == 1 and isinstance(args[0], TemporaryClass): + unwrap_me = args[0] + for attr in dir(unwrap_me): + if not attr.startswith('__'): + setattr(self, attr, getattr(unwrap_me, attr)) + else: + super(TemporaryClass, self).__init__(*args, **kwargs) + + class __metaclass__(type): + def __instancecheck__(cls, inst): + return instance_checker(inst) + + def __subclasscheck__(cls, classinfo): + value = sys.exc_info()[1] + return isinstance(value, cls) + + TemporaryClass.__name__ = instance_checker.__name__ + TemporaryClass.__doc__ = instance_checker.__doc__ + return TemporaryClass + + return wrapped + + @instance_checking_exception(EnvironmentError) + def FileNotFoundError(inst): + return getattr(inst, 'errno', object()) == errno.ENOENT + + @instance_checking_exception(EnvironmentError) + def ProcessLookupError(inst): + return getattr(inst, 'errno', object()) == errno.ESRCH + + @instance_checking_exception(EnvironmentError) + def PermissionError(inst): + return getattr(inst, 'errno', object()) in ( + errno.EACCES, errno.EPERM) + + @instance_checking_exception(EnvironmentError) + def InterruptedError(inst): + return getattr(inst, 'errno', object()) == errno.EINTR + + @instance_checking_exception(EnvironmentError) + def ChildProcessError(inst): + return getattr(inst, 'errno', object()) == errno.ECHILD + + @instance_checking_exception(EnvironmentError) + def FileExistsError(inst): + return getattr(inst, 'errno', object()) == errno.EEXIST + + # --- stdlib additions @@ -239,3 +309,24 @@ except ImportError: if _access_check(name, mode): return name return None + + +# python 3.3 +try: + from shutil import get_terminal_size +except ImportError: + def get_terminal_size(fallback=(80, 24)): + try: + import fcntl + import termios + import struct + except ImportError: + return fallback + else: + try: + # This should work on Linux. + res = struct.unpack( + 'hh', fcntl.ioctl(1, termios.TIOCGWINSZ, '1234')) + return (res[1], res[0]) + except Exception: + return fallback diff --git a/server/www/packages/packages-linux/x64/psutil/_exceptions.py b/server/www/packages/packages-linux/x64/psutil/_exceptions.py deleted file mode 100644 index 6dbbd28..0000000 --- a/server/www/packages/packages-linux/x64/psutil/_exceptions.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - - -class Error(Exception): - """Base exception class. All other psutil exceptions inherit - from this one. - """ - - def __init__(self, msg=""): - Exception.__init__(self, msg) - self.msg = msg - - def __repr__(self): - ret = "psutil.%s %s" % (self.__class__.__name__, self.msg) - return ret.strip() - - __str__ = __repr__ - - -class NoSuchProcess(Error): - """Exception raised when a process with a certain PID doesn't - or no longer exists. - """ - - def __init__(self, pid, name=None, msg=None): - Error.__init__(self, msg) - self.pid = pid - self.name = name - self.msg = msg - if msg is None: - if name: - details = "(pid=%s, name=%s)" % (self.pid, repr(self.name)) - else: - details = "(pid=%s)" % self.pid - self.msg = "process no longer exists " + details - - -class ZombieProcess(NoSuchProcess): - """Exception raised when querying a zombie process. This is - raised on macOS, BSD and Solaris only, and not always: depending - on the query the OS may be able to succeed anyway. - On Linux all zombie processes are querable (hence this is never - raised). Windows doesn't have zombie processes. - """ - - def __init__(self, pid, name=None, ppid=None, msg=None): - NoSuchProcess.__init__(self, msg) - self.pid = pid - self.ppid = ppid - self.name = name - self.msg = msg - if msg is None: - args = ["pid=%s" % pid] - if name: - args.append("name=%s" % repr(self.name)) - if ppid: - args.append("ppid=%s" % self.ppid) - details = "(%s)" % ", ".join(args) - self.msg = "process still exists but it's a zombie " + details - - -class AccessDenied(Error): - """Exception raised when permission to perform an action is denied.""" - - def __init__(self, pid=None, name=None, msg=None): - Error.__init__(self, msg) - self.pid = pid - self.name = name - self.msg = msg - if msg is None: - if (pid is not None) and (name is not None): - self.msg = "(pid=%s, name=%s)" % (pid, repr(name)) - elif (pid is not None): - self.msg = "(pid=%s)" % self.pid - else: - self.msg = "" - - -class TimeoutExpired(Error): - """Raised on Process.wait(timeout) if timeout expires and process - is still alive. - """ - - def __init__(self, seconds, pid=None, name=None): - Error.__init__(self, "timeout after %s seconds" % seconds) - self.seconds = seconds - self.pid = pid - self.name = name - if (pid is not None) and (name is not None): - self.msg += " (pid=%s, name=%s)" % (pid, repr(name)) - elif (pid is not None): - self.msg += " (pid=%s)" % self.pid diff --git a/server/www/packages/packages-linux/x64/psutil/_psaix.py b/server/www/packages/packages-linux/x64/psutil/_psaix.py index 7ba212d..79e3be1 100644 --- a/server/www/packages/packages-linux/x64/psutil/_psaix.py +++ b/server/www/packages/packages-linux/x64/psutil/_psaix.py @@ -6,31 +6,29 @@ """AIX platform implementation.""" -import errno +import functools import glob import os import re import subprocess import sys from collections import namedtuple -from socket import AF_INET from . import _common from . import _psposix from . import _psutil_aix as cext from . import _psutil_posix as cext_posix -from ._common import AF_INET6 +from ._common import conn_to_ntuple +from ._common import get_procfs_path from ._common import memoize_when_activated from ._common import NIC_DUPLEX_FULL from ._common import NIC_DUPLEX_HALF from ._common import NIC_DUPLEX_UNKNOWN -from ._common import sockfam_to_enum -from ._common import socktype_to_enum from ._common import usage_percent +from ._compat import FileNotFoundError +from ._compat import PermissionError +from ._compat import ProcessLookupError from ._compat import PY3 -from ._exceptions import AccessDenied -from ._exceptions import NoSuchProcess -from ._exceptions import ZombieProcess __extra__all__ = ["PROCFS_PATH"] @@ -42,6 +40,8 @@ __extra__all__ = ["PROCFS_PATH"] HAS_THREADS = hasattr(cext, "proc_threads") +HAS_NET_IO_COUNTERS = hasattr(cext, "net_io_counters") +HAS_PROC_IO_COUNTERS = hasattr(cext, "proc_io_counters") PAGE_SIZE = os.sysconf('SC_PAGE_SIZE') AF_LINK = cext_posix.AF_LINK @@ -79,6 +79,13 @@ proc_info_map = dict( status=6, ttynr=7) +# These objects get set on "import psutil" from the __init__.py +# file, see: https://github.com/giampaolo/psutil/issues/1402 +NoSuchProcess = None +ZombieProcess = None +AccessDenied = None +TimeoutExpired = None + # ===================================================================== # --- named tuples @@ -93,21 +100,6 @@ pfullmem = pmem scputimes = namedtuple('scputimes', ['user', 'system', 'idle', 'iowait']) # psutil.virtual_memory() svmem = namedtuple('svmem', ['total', 'available', 'percent', 'used', 'free']) -# psutil.Process.memory_maps(grouped=True) -pmmap_grouped = namedtuple('pmmap_grouped', ['path', 'rss', 'anon', 'locked']) -# psutil.Process.memory_maps(grouped=False) -pmmap_ext = namedtuple( - 'pmmap_ext', 'addr perms ' + ' '.join(pmmap_grouped._fields)) - - -# ===================================================================== -# --- utils -# ===================================================================== - - -def get_procfs_path(): - """Return updated psutil.PROCFS_PATH constant.""" - return sys.modules['psutil'].PROCFS_PATH # ===================================================================== @@ -212,7 +204,9 @@ def disk_partitions(all=False): net_if_addrs = cext_posix.net_if_addrs -net_io_counters = cext.net_io_counters + +if HAS_NET_IO_COUNTERS: + net_io_counters = cext.net_io_counters def net_connections(kind, _pid=-1): @@ -225,27 +219,17 @@ def net_connections(kind, _pid=-1): % (kind, ', '.join([repr(x) for x in cmap]))) families, types = _common.conn_tmap[kind] rawlist = cext.net_connections(_pid) - ret = set() + ret = [] for item in rawlist: fd, fam, type_, laddr, raddr, status, pid = item if fam not in families: continue if type_ not in types: continue - status = TCP_STATUSES[status] - if fam in (AF_INET, AF_INET6): - if laddr: - laddr = _common.addr(*laddr) - if raddr: - raddr = _common.addr(*raddr) - fam = sockfam_to_enum(fam) - type_ = socktype_to_enum(type_) - if _pid == -1: - nt = _common.sconn(fd, fam, type_, laddr, raddr, status, pid) - else: - nt = _common.pconn(fd, fam, type_, laddr, raddr, status) - ret.add(nt) - return list(ret) + nt = conn_to_ntuple(fd, fam, type_, laddr, raddr, status, + TCP_STATUSES, pid=pid if _pid == -1 else None) + ret.append(nt) + return ret def net_if_stats(): @@ -328,33 +312,27 @@ def wrap_exceptions(fun): """Call callable into a try/except clause and translate ENOENT, EACCES and EPERM in NoSuchProcess or AccessDenied exceptions. """ - + @functools.wraps(fun) def wrapper(self, *args, **kwargs): try: return fun(self, *args, **kwargs) - except EnvironmentError as err: - # support for private module import - if (NoSuchProcess is None or AccessDenied is None or - ZombieProcess is None): - raise + except (FileNotFoundError, ProcessLookupError): # ENOENT (no such file or directory) gets raised on open(). # ESRCH (no such process) can get raised on read() if # process is gone in meantime. - if err.errno in (errno.ENOENT, errno.ESRCH): - if not pid_exists(self.pid): - raise NoSuchProcess(self.pid, self._name) - else: - raise ZombieProcess(self.pid, self._name, self._ppid) - if err.errno in (errno.EPERM, errno.EACCES): - raise AccessDenied(self.pid, self._name) - raise + if not pid_exists(self.pid): + raise NoSuchProcess(self.pid, self._name) + else: + raise ZombieProcess(self.pid, self._name, self._ppid) + except PermissionError: + raise AccessDenied(self.pid, self._name) return wrapper class Process(object): """Wrapper class around underlying C implementation.""" - __slots__ = ["pid", "_name", "_ppid", "_procfs_path"] + __slots__ = ["pid", "_name", "_ppid", "_procfs_path", "_cache"] def __init__(self, pid): self.pid = pid @@ -363,23 +341,19 @@ class Process(object): self._procfs_path = get_procfs_path() def oneshot_enter(self): - self._proc_name_and_args.cache_activate() - self._proc_basic_info.cache_activate() - self._proc_cred.cache_activate() + self._proc_basic_info.cache_activate(self) + self._proc_cred.cache_activate(self) def oneshot_exit(self): - self._proc_name_and_args.cache_deactivate() - self._proc_basic_info.cache_deactivate() - self._proc_cred.cache_deactivate() - - @memoize_when_activated - def _proc_name_and_args(self): - return cext.proc_name_and_args(self.pid, self._procfs_path) + self._proc_basic_info.cache_deactivate(self) + self._proc_cred.cache_deactivate(self) + @wrap_exceptions @memoize_when_activated def _proc_basic_info(self): return cext.proc_basic_info(self.pid, self._procfs_path) + @wrap_exceptions @memoize_when_activated def _proc_cred(self): return cext.proc_cred(self.pid, self._procfs_path) @@ -388,22 +362,25 @@ class Process(object): def name(self): if self.pid == 0: return "swapper" - # note: this is limited to 15 characters - return self._proc_name_and_args()[0].rstrip("\x00") + # note: max 16 characters + return cext.proc_name(self.pid, self._procfs_path).rstrip("\x00") @wrap_exceptions def exe(self): # there is no way to get executable path in AIX other than to guess, # and guessing is more complex than what's in the wrapping class - exe = self.cmdline()[0] + cmdline = self.cmdline() + if not cmdline: + return '' + exe = cmdline[0] if os.path.sep in exe: # relative or absolute path if not os.path.isabs(exe): # if cwd has changed, we're out of luck - this may be wrong! exe = os.path.abspath(os.path.join(self.cwd(), exe)) if (os.path.isabs(exe) and - os.path.isfile(exe) and - os.access(exe, os.X_OK)): + os.path.isfile(exe) and + os.access(exe, os.X_OK)): return exe # not found, move to search in PATH using basename only exe = os.path.basename(exe) @@ -411,13 +388,17 @@ class Process(object): for path in os.environ["PATH"].split(":"): possible_exe = os.path.abspath(os.path.join(path, exe)) if (os.path.isfile(possible_exe) and - os.access(possible_exe, os.X_OK)): + os.access(possible_exe, os.X_OK)): return possible_exe return '' @wrap_exceptions def cmdline(self): - return self._proc_name_and_args()[1].split(' ') + return cext.proc_args(self.pid) + + @wrap_exceptions + def environ(self): + return cext.proc_environ(self.pid) @wrap_exceptions def create_time(self): @@ -503,11 +484,9 @@ class Process(object): try: result = os.readlink("%s/%s/cwd" % (procfs_path, self.pid)) return result.rstrip('/') - except OSError as err: - if err.errno == errno.ENOENT: - os.stat("%s/%s" % (procfs_path, self.pid)) # raise NSP or AD - return None - raise + except FileNotFoundError: + os.stat("%s/%s" % (procfs_path, self.pid)) # raise NSP or AD + return None @wrap_exceptions def memory_info(self): @@ -561,14 +540,15 @@ class Process(object): def wait(self, timeout=None): return _psposix.wait_pid(self.pid, timeout, self._name) - @wrap_exceptions - def io_counters(self): - try: - rc, wc, rb, wb = cext.proc_io_counters(self.pid) - except OSError: - # if process is terminated, proc_io_counters returns OSError - # instead of NSP - if not pid_exists(self.pid): - raise NoSuchProcess(self.pid, self._name) - raise - return _common.pio(rc, wc, rb, wb) + if HAS_PROC_IO_COUNTERS: + @wrap_exceptions + def io_counters(self): + try: + rc, wc, rb, wb = cext.proc_io_counters(self.pid) + except OSError: + # if process is terminated, proc_io_counters returns OSError + # instead of NSP + if not pid_exists(self.pid): + raise NoSuchProcess(self.pid, self._name) + raise + return _common.pio(rc, wc, rb, wb) diff --git a/server/www/packages/packages-linux/x64/psutil/_psbsd.py b/server/www/packages/packages-linux/x64/psutil/_psbsd.py index c2896cb..2f41dc0 100644 --- a/server/www/packages/packages-linux/x64/psutil/_psbsd.py +++ b/server/www/packages/packages-linux/x64/psutil/_psbsd.py @@ -10,26 +10,25 @@ import functools import os import xml.etree.ElementTree as ET from collections import namedtuple -from socket import AF_INET +from collections import defaultdict from . import _common from . import _psposix from . import _psutil_bsd as cext from . import _psutil_posix as cext_posix -from ._common import AF_INET6 from ._common import conn_tmap +from ._common import conn_to_ntuple from ._common import FREEBSD from ._common import memoize from ._common import memoize_when_activated from ._common import NETBSD from ._common import OPENBSD -from ._common import sockfam_to_enum -from ._common import socktype_to_enum from ._common import usage_percent +from ._compat import FileNotFoundError +from ._compat import PermissionError +from ._compat import ProcessLookupError from ._compat import which -from ._exceptions import AccessDenied -from ._exceptions import NoSuchProcess -from ._exceptions import ZombieProcess + __extra__all__ = [] @@ -136,6 +135,13 @@ kinfo_proc_map = dict( name=24, ) +# These objects get set on "import psutil" from the __init__.py +# file, see: https://github.com/giampaolo/psutil/issues/1402 +NoSuchProcess = None +ZombieProcess = None +AccessDenied = None +TimeoutExpired = None + # ===================================================================== # --- named tuples @@ -394,22 +400,8 @@ def net_connections(kind): fd, fam, type, laddr, raddr, status, pid = item # TODO: apply filter at C level if fam in families and type in types: - try: - status = TCP_STATUSES[status] - except KeyError: - # XXX: Not sure why this happens. I saw this occurring - # with IPv6 sockets opened by 'vim'. Those sockets - # have a very short lifetime so maybe the kernel - # can't initialize their status? - status = TCP_STATUSES[cext.PSUTIL_CONN_NONE] - if fam in (AF_INET, AF_INET6): - if laddr: - laddr = _common.addr(*laddr) - if raddr: - raddr = _common.addr(*raddr) - fam = sockfam_to_enum(fam) - type = socktype_to_enum(type) - nt = _common.sconn(fd, fam, type, laddr, raddr, status, pid) + nt = conn_to_ntuple(fd, fam, type, laddr, raddr, status, + TCP_STATUSES, pid) ret.add(nt) return list(ret) @@ -437,6 +429,47 @@ if FREEBSD: secsleft = minsleft * 60 return _common.sbattery(percent, secsleft, power_plugged) + def sensors_temperatures(): + "Return CPU cores temperatures if available, else an empty dict." + ret = defaultdict(list) + num_cpus = cpu_count_logical() + for cpu in range(num_cpus): + try: + current, high = cext.sensors_cpu_temperature(cpu) + if high <= 0: + high = None + name = "Core %s" % cpu + ret["coretemp"].append( + _common.shwtemp(name, current, high, high)) + except NotImplementedError: + pass + + return ret + + def cpu_freq(): + """Return frequency metrics for CPUs. As of Dec 2018 only + CPU 0 appears to be supported by FreeBSD and all other cores + match the frequency of CPU 0. + """ + ret = [] + num_cpus = cpu_count_logical() + for cpu in range(num_cpus): + try: + current, available_freq = cext.cpu_frequency(cpu) + except NotImplementedError: + continue + if available_freq: + try: + min_freq = int(available_freq.split(" ")[-1].split("/")[0]) + except(IndexError, ValueError): + min_freq = None + try: + max_freq = int(available_freq.split(" ")[0].split("/")[0]) + except(IndexError, ValueError): + max_freq = None + ret.append(_common.scpufreq(current, min_freq, max_freq)) + return ret + # ===================================================================== # --- other system functions @@ -505,6 +538,14 @@ else: pid_exists = _psposix.pid_exists +def is_zombie(pid): + try: + st = cext.proc_oneshot_info(pid)[kinfo_proc_map['status']] + return st == cext.SZOMB + except Exception: + return False + + def wrap_exceptions(fun): """Decorator which translates bare OSError exceptions into NoSuchProcess and AccessDenied. @@ -513,19 +554,19 @@ def wrap_exceptions(fun): def wrapper(self, *args, **kwargs): try: return fun(self, *args, **kwargs) - except OSError as err: + except ProcessLookupError: + if not pid_exists(self.pid): + raise NoSuchProcess(self.pid, self._name) + else: + raise ZombieProcess(self.pid, self._name, self._ppid) + except PermissionError: + raise AccessDenied(self.pid, self._name) + except OSError: if self.pid == 0: if 0 in pids(): raise AccessDenied(self.pid, self._name) else: raise - if err.errno == errno.ESRCH: - if not pid_exists(self.pid): - raise NoSuchProcess(self.pid, self._name) - else: - raise ZombieProcess(self.pid, self._name, self._ppid) - if err.errno in (errno.EPERM, errno.EACCES): - raise AccessDenied(self.pid, self._name) raise return wrapper @@ -535,30 +576,35 @@ def wrap_exceptions_procfs(inst): """Same as above, for routines relying on reading /proc fs.""" try: yield - except EnvironmentError as err: + except (ProcessLookupError, FileNotFoundError): # ENOENT (no such file or directory) gets raised on open(). # ESRCH (no such process) can get raised on read() if # process is gone in meantime. - if err.errno in (errno.ENOENT, errno.ESRCH): - if not pid_exists(inst.pid): - raise NoSuchProcess(inst.pid, inst._name) - else: - raise ZombieProcess(inst.pid, inst._name, inst._ppid) - if err.errno in (errno.EPERM, errno.EACCES): - raise AccessDenied(inst.pid, inst._name) - raise + if not pid_exists(inst.pid): + raise NoSuchProcess(inst.pid, inst._name) + else: + raise ZombieProcess(inst.pid, inst._name, inst._ppid) + except PermissionError: + raise AccessDenied(inst.pid, inst._name) class Process(object): """Wrapper class around underlying C implementation.""" - __slots__ = ["pid", "_name", "_ppid"] + __slots__ = ["pid", "_name", "_ppid", "_cache"] def __init__(self, pid): self.pid = pid self._name = None self._ppid = None + def _assert_alive(self): + """Raise NSP if the process disappeared on us.""" + # For those C function who do not raise NSP, possibly returning + # incorrect or incomplete result. + cext.proc_name(self.pid) + + @wrap_exceptions @memoize_when_activated def oneshot(self): """Retrieves multiple process info in one shot as a raw tuple.""" @@ -567,10 +613,10 @@ class Process(object): return ret def oneshot_enter(self): - self.oneshot.cache_activate() + self.oneshot.cache_activate(self) def oneshot_exit(self): - self.oneshot.cache_deactivate() + self.oneshot.cache_deactivate(self) @wrap_exceptions def name(self): @@ -612,10 +658,14 @@ class Process(object): return cext.proc_cmdline(self.pid) except OSError as err: if err.errno == errno.EINVAL: - if not pid_exists(self.pid): - raise NoSuchProcess(self.pid, self._name) - else: + if is_zombie(self.pid): raise ZombieProcess(self.pid, self._name, self._ppid) + elif not pid_exists(self.pid): + raise NoSuchProcess(self.pid, self._name, self._ppid) + else: + # XXX: this happens with unicode tests. It means the C + # routine is unable to decode invalid unicode chars. + return [] else: raise else: @@ -705,10 +755,7 @@ class Process(object): ntuple = _common.pthread(thread_id, utime, stime) retlist.append(ntuple) if OPENBSD: - # On OpenBSD the underlying C function does not raise NSP - # in case the process is gone (and the returned list may - # incomplete). - self.name() # raise NSP if the process disappeared on us + self._assert_alive() return retlist @wrap_exceptions @@ -719,29 +766,16 @@ class Process(object): if NETBSD: families, types = conn_tmap[kind] - ret = set() + ret = [] rawlist = cext.net_connections(self.pid) for item in rawlist: fd, fam, type, laddr, raddr, status, pid = item assert pid == self.pid if fam in families and type in types: - try: - status = TCP_STATUSES[status] - except KeyError: - status = TCP_STATUSES[cext.PSUTIL_CONN_NONE] - if fam in (AF_INET, AF_INET6): - if laddr: - laddr = _common.addr(*laddr) - if raddr: - raddr = _common.addr(*raddr) - fam = sockfam_to_enum(fam) - type = socktype_to_enum(type) - nt = _common.pconn(fd, fam, type, laddr, raddr, status) - ret.add(nt) - # On NetBSD the underlying C function does not raise NSP - # in case the process is gone (and the returned list may - # incomplete). - self.name() # raise NSP if the process disappeared on us + nt = conn_to_ntuple(fd, fam, type, laddr, raddr, status, + TCP_STATUSES) + ret.append(nt) + self._assert_alive() return list(ret) families, types = conn_tmap[kind] @@ -749,21 +783,13 @@ class Process(object): ret = [] for item in rawlist: fd, fam, type, laddr, raddr, status = item - if fam in (AF_INET, AF_INET6): - if laddr: - laddr = _common.addr(*laddr) - if raddr: - raddr = _common.addr(*raddr) - fam = sockfam_to_enum(fam) - type = socktype_to_enum(type) - status = TCP_STATUSES[status] - nt = _common.pconn(fd, fam, type, laddr, raddr, status) + nt = conn_to_ntuple(fd, fam, type, laddr, raddr, status, + TCP_STATUSES) ret.append(nt) + if OPENBSD: - # On OpenBSD the underlying C function does not raise NSP - # in case the process is gone (and the returned list may - # incomplete). - self.name() # raise NSP if the process disappeared on us + self._assert_alive() + return ret @wrap_exceptions @@ -800,10 +826,7 @@ class Process(object): # it into None if OPENBSD and self.pid == 0: return None # ...else it would raise EINVAL - elif NETBSD: - with wrap_exceptions_procfs(self): - return os.readlink("/proc/%s/cwd" % self.pid) - elif HAS_PROC_OPEN_FILES: + elif NETBSD or HAS_PROC_OPEN_FILES: # FreeBSD < 8 does not support functions based on # kinfo_getfile() and kinfo_getvmmap() return cext.proc_cwd(self.pid) or None @@ -839,9 +862,7 @@ class Process(object): """Return the number of file descriptors opened by this process.""" ret = cext.proc_num_fds(self.pid) if NETBSD: - # On NetBSD the underlying C function does not raise NSP - # in case the process is gone. - self.name() # raise NSP if the process disappeared on us + self._assert_alive() return ret else: num_fds = _not_implemented diff --git a/server/www/packages/packages-linux/x64/psutil/_pslinux.py b/server/www/packages/packages-linux/x64/psutil/_pslinux.py index b775d39..d29ccc8 100644 --- a/server/www/packages/packages-linux/x64/psutil/_pslinux.py +++ b/server/www/packages/packages-linux/x64/psutil/_pslinux.py @@ -25,8 +25,8 @@ from . import _common from . import _psposix from . import _psutil_linux as cext from . import _psutil_posix as cext_posix -from ._common import ENCODING -from ._common import ENCODING_ERRS +from ._common import decode +from ._common import get_procfs_path from ._common import isfile_strict from ._common import memoize from ._common import memoize_when_activated @@ -41,11 +41,10 @@ from ._common import supports_ipv6 from ._common import usage_percent from ._compat import b from ._compat import basestring -from ._compat import long +from ._compat import FileNotFoundError +from ._compat import PermissionError +from ._compat import ProcessLookupError from ._compat import PY3 -from ._exceptions import AccessDenied -from ._exceptions import NoSuchProcess -from ._exceptions import ZombieProcess if sys.version_info >= (3, 4): import enum @@ -74,6 +73,7 @@ POWER_SUPPLY_PATH = "/sys/class/power_supply" HAS_SMAPS = os.path.exists('/proc/%s/smaps' % os.getpid()) HAS_PRLIMIT = hasattr(cext, "linux_prlimit") HAS_PROC_IO_PRIORITY = hasattr(cext, "proc_ioprio_get") +HAS_CPU_AFFINITY = hasattr(cext, "proc_cpu_affinity_get") _DEFAULT = object() # RLIMIT_* constants, not guaranteed to be present on all kernels @@ -161,6 +161,13 @@ TCP_STATUSES = { "0B": _common.CONN_CLOSING } +# These objects get set on "import psutil" from the __init__.py +# file, see: https://github.com/giampaolo/psutil/issues/1402 +NoSuchProcess = None +ZombieProcess = None +AccessDenied = None +TimeoutExpired = None + # ===================================================================== # --- named tuples @@ -197,6 +204,10 @@ pmmap_ext = namedtuple( pio = namedtuple('pio', ['read_count', 'write_count', 'read_bytes', 'write_bytes', 'read_chars', 'write_chars']) +# psutil.Process.cpu_times() +pcputimes = namedtuple('pcputimes', + ['user', 'system', 'children_user', 'children_system', + 'iowait']) # ===================================================================== @@ -204,19 +215,6 @@ pio = namedtuple('pio', ['read_count', 'write_count', # ===================================================================== -if PY3: - def decode(s): - return s.decode(encoding=ENCODING, errors=ENCODING_ERRS) -else: - def decode(s): - return s - - -def get_procfs_path(): - """Return updated psutil.PROCFS_PATH constant.""" - return sys.modules['psutil'].PROCFS_PATH - - def readlink(path): """Wrapper around os.readlink().""" assert isinstance(path, basestring), path @@ -623,6 +621,17 @@ def cpu_count_logical(): def cpu_count_physical(): """Return the number of physical cores in the system.""" + # Method #1 + core_ids = set() + for path in glob.glob( + "/sys/devices/system/cpu/cpu[0-9]*/topology/core_id"): + with open_binary(path) as f: + core_ids.add(int(f.read())) + result = len(core_ids) + if result != 0: + return result + + # Method #2 mapping = {} current_info = {} with open_binary('%s/cpuinfo' % get_procfs_path()) as f: @@ -642,8 +651,8 @@ def cpu_count_physical(): key, value = line.split(b'\t:', 1) current_info[key] = int(value) - # mimic os.cpu_count() - return sum(mapping.values()) or None + result = sum(mapping.values()) + return result or None # mimic os.cpu_count() def cpu_stats(): @@ -667,30 +676,26 @@ def cpu_stats(): ctx_switches, interrupts, soft_interrupts, syscalls) -if os.path.exists("/sys/devices/system/cpu/cpufreq") or \ +if os.path.exists("/sys/devices/system/cpu/cpufreq/policy0") or \ os.path.exists("/sys/devices/system/cpu/cpu0/cpufreq"): def cpu_freq(): """Return frequency metrics for all CPUs. Contrarily to other OSes, Linux updates these values in real-time. """ - # scaling_* files seem preferable to cpuinfo_*, see: - # http://unix.stackexchange.com/a/87537/168884 - ret = [] - ls = glob.glob("/sys/devices/system/cpu/cpufreq/policy*") - if ls: - # Sort the list so that '10' comes after '2'. This should - # ensure the CPU order is consistent with other CPU functions - # having a 'percpu' argument and returning results for multiple - # CPUs (cpu_times(), cpu_percent(), cpu_times_percent()). - ls.sort(key=lambda x: int(os.path.basename(x)[6:])) - else: - # https://github.com/giampaolo/psutil/issues/981 - ls = glob.glob("/sys/devices/system/cpu/cpu[0-9]*/cpufreq") - ls.sort(key=lambda x: int(re.search('[0-9]+', x).group(0))) + def get_path(num): + for p in ("/sys/devices/system/cpu/cpufreq/policy%s" % num, + "/sys/devices/system/cpu/cpu%s/cpufreq" % num): + if os.path.exists(p): + return p - pjoin = os.path.join - for path in ls: + ret = [] + for n in range(cpu_count_logical()): + path = get_path(n) + if not path: + continue + + pjoin = os.path.join curr = cat(pjoin(path, "scaling_cur_freq"), fallback=None) if curr is None: # Likely an old RedHat, see: @@ -715,9 +720,15 @@ elif os.path.exists("/proc/cpuinfo"): for line in f: if line.lower().startswith(b'cpu mhz'): key, value = line.split(b'\t:', 1) - ret.append(_common.scpufreq(float(value), None, None)) + ret.append(_common.scpufreq(float(value), 0., 0.)) return ret +else: + def cpu_freq(): + """Dummy implementation when none of the above files are present. + """ + return [] + # ===================================================================== # --- network @@ -769,17 +780,16 @@ class Connections: for fd in os.listdir("%s/%s/fd" % (self._procfs_path, pid)): try: inode = readlink("%s/%s/fd/%s" % (self._procfs_path, pid, fd)) - except OSError as err: + except (FileNotFoundError, ProcessLookupError): # ENOENT == file which is gone in the meantime; # os.stat('/proc/%s' % self.pid) will be done later # to force NSP (if it's the case) - if err.errno in (errno.ENOENT, errno.ESRCH): - continue - elif err.errno == errno.EINVAL: + continue + except OSError as err: + if err.errno == errno.EINVAL: # not a link continue - else: - raise + raise else: if inode.startswith('socket:['): # the process is using a socket @@ -792,7 +802,7 @@ class Connections: for pid in pids(): try: inodes.update(self.get_proc_inodes(pid)) - except OSError as err: + except (FileNotFoundError, ProcessLookupError, PermissionError): # os.listdir() is gonna raise a lot of access denied # exceptions in case of unprivileged user; that's fine # as we'll just end up returning a connection with PID @@ -800,9 +810,7 @@ class Connections: # Both netstat -an and lsof does the same so it's # unlikely we can do any better. # ENOENT just means a PID disappeared on us. - if err.errno not in ( - errno.ENOENT, errno.ESRCH, errno.EPERM, errno.EACCES): - raise + continue return inodes @staticmethod @@ -930,7 +938,7 @@ class Connections: path = tokens[-1] else: path = "" - type_ = int(type_) + type_ = _common.socktype_to_enum(int(type_)) # XXX: determining the remote endpoint of a # UNIX socket on Linux is not possible, see: # https://serverfault.com/questions/252723/ @@ -1155,13 +1163,13 @@ def disk_partitions(all=False): fstypes.add("zfs") # See: https://github.com/giampaolo/psutil/issues/1307 - if procfs_path == "/proc": - mtab_path = os.path.realpath("/etc/mtab") + if procfs_path == "/proc" and os.path.isfile('/etc/mtab'): + mounts_path = os.path.realpath("/etc/mtab") else: - mtab_path = os.path.realpath("%s/self/mounts" % procfs_path) + mounts_path = os.path.realpath("%s/self/mounts" % procfs_path) retlist = [] - partitions = cext.disk_partitions(mtab_path) + partitions = cext.disk_partitions(mounts_path) for partition in partitions: device, mountpoint, fstype, opts = partition if device == 'none': @@ -1487,11 +1495,10 @@ def ppid_map(): try: with open_binary("%s/%s/stat" % (procfs_path, pid)) as f: data = f.read() - except EnvironmentError as err: + except (FileNotFoundError, ProcessLookupError): # Note: we should be able to access /stat for all processes # aka it's unlikely we'll bump into EPERM, which is good. - if err.errno not in (errno.ENOENT, errno.ESRCH): - raise + pass else: rpar = data.rfind(b')') dset = data[rpar + 2:].split() @@ -1508,16 +1515,12 @@ def wrap_exceptions(fun): def wrapper(self, *args, **kwargs): try: return fun(self, *args, **kwargs) - except EnvironmentError as err: - if err.errno in (errno.EPERM, errno.EACCES): - raise AccessDenied(self.pid, self._name) - # ESRCH (no such process) can be raised on read() if - # process is gone in the meantime. - if err.errno == errno.ESRCH: - raise NoSuchProcess(self.pid, self._name) - # ENOENT (no such file or directory) can be raised on open(). - if err.errno == errno.ENOENT and not os.path.exists("%s/%s" % ( - self._procfs_path, self.pid)): + except PermissionError: + raise AccessDenied(self.pid, self._name) + except ProcessLookupError: + raise NoSuchProcess(self.pid, self._name) + except FileNotFoundError: + if not os.path.exists("%s/%s" % (self._procfs_path, self.pid)): raise NoSuchProcess(self.pid, self._name) # Note: zombies will keep existing under /proc until they're # gone so there's no way to distinguish them in here. @@ -1528,7 +1531,7 @@ def wrap_exceptions(fun): class Process(object): """Linux process implementation.""" - __slots__ = ["pid", "_name", "_ppid", "_procfs_path"] + __slots__ = ["pid", "_name", "_ppid", "_procfs_path", "_cache"] def __init__(self, pid): self.pid = pid @@ -1536,13 +1539,20 @@ class Process(object): self._ppid = None self._procfs_path = get_procfs_path() + def _assert_alive(self): + """Raise NSP if the process disappeared on us.""" + # For those C function who do not raise NSP, possibly returning + # incorrect or incomplete result. + os.stat('%s/%s' % (self._procfs_path, self.pid)) + + @wrap_exceptions @memoize_when_activated def _parse_stat_file(self): - """Parse /proc/{pid}/stat file. Return a list of fields where - process name is in position 0. + """Parse /proc/{pid}/stat file and return a dict with various + process info. Using "man proc" as a reference: where "man proc" refers to - position N, always substract 2 (e.g starttime pos 22 in - 'man proc' == pos 20 in the list returned here). + position N always substract 3 (e.g ppid position 4 in + 'man proc' == position 1 in here). The return value is cached in case oneshot() ctx manager is in use. """ @@ -1553,9 +1563,24 @@ class Process(object): # the first occurrence of "(" and the last occurence of ")". rpar = data.rfind(b')') name = data[data.find(b'(') + 1:rpar] - others = data[rpar + 2:].split() - return [name] + others + fields = data[rpar + 2:].split() + ret = {} + ret['name'] = name + ret['status'] = fields[0] + ret['ppid'] = fields[1] + ret['ttynr'] = fields[4] + ret['utime'] = fields[11] + ret['stime'] = fields[12] + ret['children_utime'] = fields[13] + ret['children_stime'] = fields[14] + ret['create_time'] = fields[19] + ret['cpu_num'] = fields[36] + ret['blkio_ticks'] = fields[39] # aka 'delayacct_blkio_ticks' + + return ret + + @wrap_exceptions @memoize_when_activated def _read_status_file(self): """Read /proc/{pid}/stat file and return its content. @@ -1565,6 +1590,7 @@ class Process(object): with open_binary("%s/%s/status" % (self._procfs_path, self.pid)) as f: return f.read() + @wrap_exceptions @memoize_when_activated def _read_smaps_file(self): with open_binary("%s/%s/smaps" % (self._procfs_path, self.pid), @@ -1572,18 +1598,18 @@ class Process(object): return f.read().strip() def oneshot_enter(self): - self._parse_stat_file.cache_activate() - self._read_status_file.cache_activate() - self._read_smaps_file.cache_activate() + self._parse_stat_file.cache_activate(self) + self._read_status_file.cache_activate(self) + self._read_smaps_file.cache_activate(self) def oneshot_exit(self): - self._parse_stat_file.cache_deactivate() - self._read_status_file.cache_deactivate() - self._read_smaps_file.cache_deactivate() + self._parse_stat_file.cache_deactivate(self) + self._read_status_file.cache_deactivate(self) + self._read_smaps_file.cache_deactivate(self) @wrap_exceptions def name(self): - name = self._parse_stat_file()[0] + name = self._parse_stat_file()['name'] if PY3: name = decode(name) # XXX - gets changed later and probably needs refactoring @@ -1592,21 +1618,19 @@ class Process(object): def exe(self): try: return readlink("%s/%s/exe" % (self._procfs_path, self.pid)) - except OSError as err: - if err.errno in (errno.ENOENT, errno.ESRCH): - # no such file error; might be raised also if the - # path actually exists for system processes with - # low pids (about 0-20) - if os.path.lexists("%s/%s" % (self._procfs_path, self.pid)): - return "" + except (FileNotFoundError, ProcessLookupError): + # no such file error; might be raised also if the + # path actually exists for system processes with + # low pids (about 0-20) + if os.path.lexists("%s/%s" % (self._procfs_path, self.pid)): + return "" + else: + if not pid_exists(self.pid): + raise NoSuchProcess(self.pid, self._name) else: - if not pid_exists(self.pid): - raise NoSuchProcess(self.pid, self._name) - else: - raise ZombieProcess(self.pid, self._name, self._ppid) - if err.errno in (errno.EPERM, errno.EACCES): - raise AccessDenied(self.pid, self._name) - raise + raise ZombieProcess(self.pid, self._name, self._ppid) + except PermissionError: + raise AccessDenied(self.pid, self._name) @wrap_exceptions def cmdline(self): @@ -1625,7 +1649,7 @@ class Process(object): sep = '\x00' if data.endswith('\x00') else ' ' if data.endswith(sep): data = data[:-1] - return [x for x in data.split(sep)] + return data.split(sep) @wrap_exceptions def environ(self): @@ -1635,13 +1659,14 @@ class Process(object): @wrap_exceptions def terminal(self): - tty_nr = int(self._parse_stat_file()[5]) + tty_nr = int(self._parse_stat_file()['ttynr']) tmap = _psposix.get_terminal_map() try: return tmap[tty_nr] except KeyError: return None + # May not be available on old kernels. if os.path.exists('/proc/%s/io' % os.getpid()): @wrap_exceptions def io_counters(self): @@ -1673,24 +1698,21 @@ class Process(object): except KeyError as err: raise ValueError("%r field was not found in %s; found fields " "are %r" % (err[0], fname, fields)) - else: - def io_counters(self): - raise NotImplementedError("couldn't find /proc/%s/io (kernel " - "too old?)" % self.pid) @wrap_exceptions def cpu_times(self): values = self._parse_stat_file() - utime = float(values[12]) / CLOCK_TICKS - stime = float(values[13]) / CLOCK_TICKS - children_utime = float(values[14]) / CLOCK_TICKS - children_stime = float(values[15]) / CLOCK_TICKS - return _common.pcputimes(utime, stime, children_utime, children_stime) + utime = float(values['utime']) / CLOCK_TICKS + stime = float(values['stime']) / CLOCK_TICKS + children_utime = float(values['children_utime']) / CLOCK_TICKS + children_stime = float(values['children_stime']) / CLOCK_TICKS + iowait = float(values['blkio_ticks']) / CLOCK_TICKS + return pcputimes(utime, stime, children_utime, children_stime, iowait) @wrap_exceptions def cpu_num(self): """What CPU the process is on.""" - return int(self._parse_stat_file()[37]) + return int(self._parse_stat_file()['cpu_num']) @wrap_exceptions def wait(self, timeout=None): @@ -1698,14 +1720,14 @@ class Process(object): @wrap_exceptions def create_time(self): - values = self._parse_stat_file() + ctime = float(self._parse_stat_file()['create_time']) # According to documentation, starttime is in field 21 and the # unit is jiffies (clock ticks). # We first divide it for clock ticks and then add uptime returning # seconds since the epoch, in UTC. # Also use cached value if available. bt = BOOT_TIME or boot_time() - return (float(values[20]) / CLOCK_TICKS) + bt + return (ctime / CLOCK_TICKS) + bt @wrap_exceptions def memory_info(self): @@ -1767,6 +1789,9 @@ class Process(object): """Return process's mapped memory regions as a list of named tuples. Fields are explained in 'man proc'; here is an updated (Apr 2012) version: http://goo.gl/fmebo + + /proc/{PID}/smaps does not exist on kernels < 2.6.14 or if + CONFIG_MMU kernel configuration option is not enabled. """ def get_blocks(lines, current_block): data = {} @@ -1827,25 +1852,16 @@ class Process(object): )) return ls - else: # pragma: no cover - def memory_maps(self): - raise NotImplementedError( - "/proc/%s/smaps does not exist on kernels < 2.6.14 or " - "if CONFIG_MMU kernel configuration option is not " - "enabled." % self.pid) - @wrap_exceptions def cwd(self): try: return readlink("%s/%s/cwd" % (self._procfs_path, self.pid)) - except OSError as err: + except (FileNotFoundError, ProcessLookupError): # https://github.com/giampaolo/psutil/issues/986 - if err.errno in (errno.ENOENT, errno.ESRCH): - if not pid_exists(self.pid): - raise NoSuchProcess(self.pid, self._name) - else: - raise ZombieProcess(self.pid, self._name, self._ppid) - raise + if not pid_exists(self.pid): + raise NoSuchProcess(self.pid, self._name) + else: + raise ZombieProcess(self.pid, self._name, self._ppid) @wrap_exceptions def num_ctx_switches(self, @@ -1881,13 +1897,11 @@ class Process(object): try: with open_binary(fname) as f: st = f.read().strip() - except IOError as err: - if err.errno == errno.ENOENT: - # no such file or directory; it means thread - # disappeared on us - hit_enoent = True - continue - raise + except FileNotFoundError: + # no such file or directory; it means thread + # disappeared on us + hit_enoent = True + continue # ignore the first two values ("pid (exe)") st = st[st.find(b')') + 2:] values = st.split(b' ') @@ -1896,8 +1910,7 @@ class Process(object): ntuple = _common.pthread(int(thread_id), utime, stime) retlist.append(ntuple) if hit_enoent: - # raise NSP if the process disappeared on us - os.stat('%s/%s' % (self._procfs_path, self.pid)) + self._assert_alive() return retlist @wrap_exceptions @@ -1913,38 +1926,41 @@ class Process(object): def nice_set(self, value): return cext_posix.setpriority(self.pid, value) - @wrap_exceptions - def cpu_affinity_get(self): - return cext.proc_cpu_affinity_get(self.pid) + # starting from CentOS 6. + if HAS_CPU_AFFINITY: - def _get_eligible_cpus( - self, _re=re.compile(br"Cpus_allowed_list:\t(\d+)-(\d+)")): - # See: https://github.com/giampaolo/psutil/issues/956 - data = self._read_status_file() - match = _re.findall(data) - if match: - return list(range(int(match[0][0]), int(match[0][1]) + 1)) - else: - return list(range(len(per_cpu_times()))) + @wrap_exceptions + def cpu_affinity_get(self): + return cext.proc_cpu_affinity_get(self.pid) - @wrap_exceptions - def cpu_affinity_set(self, cpus): - try: - cext.proc_cpu_affinity_set(self.pid, cpus) - except (OSError, ValueError) as err: - if isinstance(err, ValueError) or err.errno == errno.EINVAL: - eligible_cpus = self._get_eligible_cpus() - all_cpus = tuple(range(len(per_cpu_times()))) - for cpu in cpus: - if cpu not in all_cpus: - raise ValueError( - "invalid CPU number %r; choose between %s" % ( - cpu, eligible_cpus)) - if cpu not in eligible_cpus: - raise ValueError( - "CPU number %r is not eligible; choose " - "between %s" % (cpu, eligible_cpus)) - raise + def _get_eligible_cpus( + self, _re=re.compile(br"Cpus_allowed_list:\t(\d+)-(\d+)")): + # See: https://github.com/giampaolo/psutil/issues/956 + data = self._read_status_file() + match = _re.findall(data) + if match: + return list(range(int(match[0][0]), int(match[0][1]) + 1)) + else: + return list(range(len(per_cpu_times()))) + + @wrap_exceptions + def cpu_affinity_set(self, cpus): + try: + cext.proc_cpu_affinity_set(self.pid, cpus) + except (OSError, ValueError) as err: + if isinstance(err, ValueError) or err.errno == errno.EINVAL: + eligible_cpus = self._get_eligible_cpus() + all_cpus = tuple(range(len(per_cpu_times()))) + for cpu in cpus: + if cpu not in all_cpus: + raise ValueError( + "invalid CPU number %r; choose between %s" % ( + cpu, eligible_cpus)) + if cpu not in eligible_cpus: + raise ValueError( + "CPU number %r is not eligible; choose " + "between %s" % (cpu, eligible_cpus)) + raise # only starting from kernel 2.6.13 if HAS_PROC_IO_PRIORITY: @@ -1958,35 +1974,12 @@ class Process(object): @wrap_exceptions def ionice_set(self, ioclass, value): - if value is not None: - if not PY3 and not isinstance(value, (int, long)): - msg = "value argument is not an integer (gor %r)" % value - raise TypeError(msg) - if not 0 <= value <= 7: - raise ValueError( - "value argument range expected is between 0 and 7") - - if ioclass in (IOPRIO_CLASS_NONE, None): - if value: - msg = "can't specify value with IOPRIO_CLASS_NONE " \ - "(got %r)" % value - raise ValueError(msg) - ioclass = IOPRIO_CLASS_NONE + if value is None: value = 0 - elif ioclass == IOPRIO_CLASS_IDLE: - if value: - msg = "can't specify value with IOPRIO_CLASS_IDLE " \ - "(got %r)" % value - raise ValueError(msg) - value = 0 - elif ioclass in (IOPRIO_CLASS_RT, IOPRIO_CLASS_BE): - if value is None: - # TODO: add comment explaining why this is 4 (?) - value = 4 - else: - # otherwise we would get OSError(EVINAL) - raise ValueError("invalid ioclass argument %r" % ioclass) - + if value and ioclass in (IOPRIO_CLASS_IDLE, IOPRIO_CLASS_NONE): + raise ValueError("%r ioclass accepts no value" % ioclass) + if value < 0 or value > 7: + raise ValueError("value not in 0-7 range") return cext.proc_ioprio_set(self.pid, ioclass, value) if HAS_PRLIMIT: @@ -2020,7 +2013,7 @@ class Process(object): @wrap_exceptions def status(self): - letter = self._parse_stat_file()[1] + letter = self._parse_stat_file()['status'] if PY3: letter = letter.decode() # XXX is '?' legit? (we're not supposed to return it anyway) @@ -2035,16 +2028,15 @@ class Process(object): file = "%s/%s/fd/%s" % (self._procfs_path, self.pid, fd) try: path = readlink(file) - except OSError as err: + except (FileNotFoundError, ProcessLookupError): # ENOENT == file which is gone in the meantime - if err.errno in (errno.ENOENT, errno.ESRCH): - hit_enoent = True - continue - elif err.errno == errno.EINVAL: + hit_enoent = True + continue + except OSError as err: + if err.errno == errno.EINVAL: # not a link continue - else: - raise + raise else: # If path is not an absolute there's no way to tell # whether it's a regular file or not, so we skip it. @@ -2058,29 +2050,23 @@ class Process(object): with open_binary(file) as f: pos = int(f.readline().split()[1]) flags = int(f.readline().split()[1], 8) - except IOError as err: - if err.errno == errno.ENOENT: - # fd gone in the meantime; does not - # necessarily mean the process disappeared - # on us. - hit_enoent = True - else: - raise + except FileNotFoundError: + # fd gone in the meantime; process may + # still be alive + hit_enoent = True else: mode = file_flags_to_mode(flags) ntuple = popenfile( path, int(fd), int(pos), mode, flags) retlist.append(ntuple) if hit_enoent: - # raise NSP if the process disappeared on us - os.stat('%s/%s' % (self._procfs_path, self.pid)) + self._assert_alive() return retlist @wrap_exceptions def connections(self, kind='inet'): ret = _connections.retrieve(kind, self.pid) - # raise NSP if the process disappeared on us - os.stat('%s/%s' % (self._procfs_path, self.pid)) + self._assert_alive() return ret @wrap_exceptions @@ -2089,7 +2075,7 @@ class Process(object): @wrap_exceptions def ppid(self): - return int(self._parse_stat_file()[2]) + return int(self._parse_stat_file()['ppid']) @wrap_exceptions def uids(self, _uids_re=re.compile(br'Uid:\t(\d+)\t(\d+)\t(\d+)')): diff --git a/server/www/packages/packages-linux/x64/psutil/_psosx.py b/server/www/packages/packages-linux/x64/psutil/_psosx.py index 94e22bc..7f28447 100644 --- a/server/www/packages/packages-linux/x64/psutil/_psosx.py +++ b/server/www/packages/packages-linux/x64/psutil/_psosx.py @@ -8,24 +8,20 @@ import contextlib import errno import functools import os -from socket import AF_INET from collections import namedtuple from . import _common from . import _psposix from . import _psutil_osx as cext from . import _psutil_posix as cext_posix -from ._common import AF_INET6 from ._common import conn_tmap +from ._common import conn_to_ntuple from ._common import isfile_strict from ._common import memoize_when_activated from ._common import parse_environ_block -from ._common import sockfam_to_enum -from ._common import socktype_to_enum +from ._compat import PermissionError +from ._compat import ProcessLookupError from ._common import usage_percent -from ._exceptions import AccessDenied -from ._exceptions import NoSuchProcess -from ._exceptions import ZombieProcess __extra__all__ = [] @@ -87,6 +83,13 @@ pidtaskinfo_map = dict( volctxsw=7, ) +# These objects get set on "import psutil" from the __init__.py +# file, see: https://github.com/giampaolo/psutil/issues/1402 +NoSuchProcess = None +ZombieProcess = None +AccessDenied = None +TimeoutExpired = None + # ===================================================================== # --- named tuples @@ -103,13 +106,6 @@ svmem = namedtuple( pmem = namedtuple('pmem', ['rss', 'vms', 'pfaults', 'pageins']) # psutil.Process.memory_full_info() pfullmem = namedtuple('pfullmem', pmem._fields + ('uss', )) -# psutil.Process.memory_maps(grouped=True) -pmmap_grouped = namedtuple( - 'pmmap_grouped', - 'path rss private swapped dirtied ref_count shadow_depth') -# psutil.Process.memory_maps(grouped=False) -pmmap_ext = namedtuple( - 'pmmap_ext', 'addr perms ' + ' '.join(pmmap_grouped._fields)) # ===================================================================== @@ -340,12 +336,10 @@ def wrap_exceptions(fun): def wrapper(self, *args, **kwargs): try: return fun(self, *args, **kwargs) - except OSError as err: - if err.errno == errno.ESRCH: - raise NoSuchProcess(self.pid, self._name) - if err.errno in (errno.EPERM, errno.EACCES): - raise AccessDenied(self.pid, self._name) - raise + except ProcessLookupError: + raise NoSuchProcess(self.pid, self._name) + except PermissionError: + raise AccessDenied(self.pid, self._name) except cext.ZombieProcessError: raise ZombieProcess(self.pid, self._name, self._ppid) return wrapper @@ -380,13 +374,14 @@ def catch_zombie(proc): class Process(object): """Wrapper class around underlying C implementation.""" - __slots__ = ["pid", "_name", "_ppid"] + __slots__ = ["pid", "_name", "_ppid", "_cache"] def __init__(self, pid): self.pid = pid self._name = None self._ppid = None + @wrap_exceptions @memoize_when_activated def _get_kinfo_proc(self): # Note: should work with all PIDs without permission issues. @@ -394,6 +389,7 @@ class Process(object): assert len(ret) == len(kinfo_proc_map) return ret + @wrap_exceptions @memoize_when_activated def _get_pidtaskinfo(self): # Note: should work for PIDs owned by user only. @@ -403,12 +399,12 @@ class Process(object): return ret def oneshot_enter(self): - self._get_kinfo_proc.cache_activate() - self._get_pidtaskinfo.cache_activate() + self._get_kinfo_proc.cache_activate(self) + self._get_pidtaskinfo.cache_activate(self) def oneshot_exit(self): - self._get_kinfo_proc.cache_deactivate() - self._get_pidtaskinfo.cache_deactivate() + self._get_kinfo_proc.cache_deactivate(self) + self._get_pidtaskinfo.cache_deactivate(self) @wrap_exceptions def name(self): @@ -530,15 +526,8 @@ class Process(object): ret = [] for item in rawlist: fd, fam, type, laddr, raddr, status = item - status = TCP_STATUSES[status] - fam = sockfam_to_enum(fam) - type = socktype_to_enum(type) - if fam in (AF_INET, AF_INET6): - if laddr: - laddr = _common.addr(*laddr) - if raddr: - raddr = _common.addr(*raddr) - nt = _common.pconn(fd, fam, type, laddr, raddr, status) + nt = conn_to_ntuple(fd, fam, type, laddr, raddr, status, + TCP_STATUSES) ret.append(nt) return ret @@ -577,7 +566,3 @@ class Process(object): ntuple = _common.pthread(thread_id, utime, stime) retlist.append(ntuple) return retlist - - @wrap_exceptions - def memory_maps(self): - return cext.proc_memory_maps(self.pid) diff --git a/server/www/packages/packages-linux/x64/psutil/_psposix.py b/server/www/packages/packages-linux/x64/psutil/_psposix.py index 9c3fac2..2457022 100644 --- a/server/www/packages/packages-linux/x64/psutil/_psposix.py +++ b/server/www/packages/packages-linux/x64/psutil/_psposix.py @@ -4,7 +4,6 @@ """Routines common to all posix systems.""" -import errno import glob import os import sys @@ -13,14 +12,23 @@ import time from ._common import memoize from ._common import sdiskusage from ._common import usage_percent +from ._compat import ChildProcessError +from ._compat import FileNotFoundError +from ._compat import InterruptedError +from ._compat import PermissionError +from ._compat import ProcessLookupError from ._compat import PY3 from ._compat import unicode -from ._exceptions import TimeoutExpired __all__ = ['pid_exists', 'wait_pid', 'disk_usage', 'get_terminal_map'] +# This object gets set on "import psutil" from the __init__.py +# file, see: https://github.com/giampaolo/psutil/issues/1402 +TimeoutExpired = None + + def pid_exists(pid): """Check whether pid exists in the current process table.""" if pid == 0: @@ -32,19 +40,13 @@ def pid_exists(pid): return True try: os.kill(pid, 0) - except OSError as err: - if err.errno == errno.ESRCH: - # ESRCH == No such process - return False - elif err.errno == errno.EPERM: - # EPERM clearly means there's a process to deny access to - return True - else: - # According to "man 2 kill" possible error values are - # (EINVAL, EPERM, ESRCH) therefore we should never get - # here. If we do let's be explicit in considering this - # an error. - raise err + except ProcessLookupError: + return False + except PermissionError: + # EPERM clearly means there's a process to deny access to + return True + # According to "man 2 kill" possible error values are + # (EINVAL, EPERM, ESRCH) else: return True @@ -80,24 +82,20 @@ def wait_pid(pid, timeout=None, proc_name=None): while True: try: retpid, status = waitcall() - except OSError as err: - if err.errno == errno.EINTR: - delay = check_timeout(delay) - continue - elif err.errno == errno.ECHILD: - # This has two meanings: - # - pid is not a child of os.getpid() in which case - # we keep polling until it's gone - # - pid never existed in the first place - # In both cases we'll eventually return None as we - # can't determine its exit status code. - while True: - if pid_exists(pid): - delay = check_timeout(delay) - else: - return - else: - raise + except InterruptedError: + delay = check_timeout(delay) + except ChildProcessError: + # This has two meanings: + # - pid is not a child of os.getpid() in which case + # we keep polling until it's gone + # - pid never existed in the first place + # In both cases we'll eventually return None as we + # can't determine its exit status code. + while True: + if pid_exists(pid): + delay = check_timeout(delay) + else: + return else: if retpid == 0: # WNOHANG was used, pid is still running @@ -176,7 +174,6 @@ def get_terminal_map(): assert name not in ret, name try: ret[os.stat(name).st_rdev] = name - except OSError as err: - if err.errno != errno.ENOENT: - raise + except FileNotFoundError: + pass return ret diff --git a/server/www/packages/packages-linux/x64/psutil/_pssunos.py b/server/www/packages/packages-linux/x64/psutil/_pssunos.py index e2f33a3..2aa2a86 100644 --- a/server/www/packages/packages-linux/x64/psutil/_pssunos.py +++ b/server/www/packages/packages-linux/x64/psutil/_pssunos.py @@ -5,6 +5,7 @@ """Sun OS Solaris platform implementation.""" import errno +import functools import os import socket import subprocess @@ -17,16 +18,17 @@ from . import _psposix from . import _psutil_posix as cext_posix from . import _psutil_sunos as cext from ._common import AF_INET6 +from ._common import get_procfs_path from ._common import isfile_strict from ._common import memoize_when_activated from ._common import sockfam_to_enum from ._common import socktype_to_enum from ._common import usage_percent from ._compat import b +from ._compat import FileNotFoundError +from ._compat import PermissionError +from ._compat import ProcessLookupError from ._compat import PY3 -from ._exceptions import AccessDenied -from ._exceptions import NoSuchProcess -from ._exceptions import ZombieProcess __extra__all__ = ["CONN_IDLE", "CONN_BOUND", "PROCFS_PATH"] @@ -85,6 +87,13 @@ proc_info_map = dict( gid=10, egid=11) +# These objects get set on "import psutil" from the __init__.py +# file, see: https://github.com/giampaolo/psutil/issues/1402 +NoSuchProcess = None +ZombieProcess = None +AccessDenied = None +TimeoutExpired = None + # ===================================================================== # --- named tuples @@ -109,16 +118,6 @@ pmmap_ext = namedtuple( 'pmmap_ext', 'addr perms ' + ' '.join(pmmap_grouped._fields)) -# ===================================================================== -# --- utils -# ===================================================================== - - -def get_procfs_path(): - """Return updated psutil.PROCFS_PATH constant.""" - return sys.modules['psutil'].PROCFS_PATH - - # ===================================================================== # --- memory # ===================================================================== @@ -266,6 +265,7 @@ def net_connections(kind, _pid=-1): continue if type_ not in types: continue + # TODO: refactor and use _common.conn_to_ntuple. if fam in (AF_INET, AF_INET6): if laddr: laddr = _common.addr(*laddr) @@ -341,26 +341,26 @@ def wrap_exceptions(fun): """Call callable into a try/except clause and translate ENOENT, EACCES and EPERM in NoSuchProcess or AccessDenied exceptions. """ - + @functools.wraps(fun) def wrapper(self, *args, **kwargs): try: return fun(self, *args, **kwargs) - except EnvironmentError as err: + except (FileNotFoundError, ProcessLookupError): + # ENOENT (no such file or directory) gets raised on open(). + # ESRCH (no such process) can get raised on read() if + # process is gone in meantime. + if not pid_exists(self.pid): + raise NoSuchProcess(self.pid, self._name) + else: + raise ZombieProcess(self.pid, self._name, self._ppid) + except PermissionError: + raise AccessDenied(self.pid, self._name) + except OSError: if self.pid == 0: if 0 in pids(): raise AccessDenied(self.pid, self._name) else: raise - # ENOENT (no such file or directory) gets raised on open(). - # ESRCH (no such process) can get raised on read() if - # process is gone in meantime. - if err.errno in (errno.ENOENT, errno.ESRCH): - if not pid_exists(self.pid): - raise NoSuchProcess(self.pid, self._name) - else: - raise ZombieProcess(self.pid, self._name, self._ppid) - if err.errno in (errno.EPERM, errno.EACCES): - raise AccessDenied(self.pid, self._name) raise return wrapper @@ -368,7 +368,7 @@ def wrap_exceptions(fun): class Process(object): """Wrapper class around underlying C implementation.""" - __slots__ = ["pid", "_name", "_ppid", "_procfs_path"] + __slots__ = ["pid", "_name", "_ppid", "_procfs_path", "_cache"] def __init__(self, pid): self.pid = pid @@ -376,32 +376,38 @@ class Process(object): self._ppid = None self._procfs_path = get_procfs_path() + def _assert_alive(self): + """Raise NSP if the process disappeared on us.""" + # For those C function who do not raise NSP, possibly returning + # incorrect or incomplete result. + os.stat('%s/%s' % (self._procfs_path, self.pid)) + def oneshot_enter(self): - self._proc_name_and_args.cache_activate() - self._proc_basic_info.cache_activate() - self._proc_cred.cache_activate() + self._proc_name_and_args.cache_activate(self) + self._proc_basic_info.cache_activate(self) + self._proc_cred.cache_activate(self) def oneshot_exit(self): - self._proc_name_and_args.cache_deactivate() - self._proc_basic_info.cache_deactivate() - self._proc_cred.cache_deactivate() + self._proc_name_and_args.cache_deactivate(self) + self._proc_basic_info.cache_deactivate(self) + self._proc_cred.cache_deactivate(self) + @wrap_exceptions @memoize_when_activated def _proc_name_and_args(self): return cext.proc_name_and_args(self.pid, self._procfs_path) + @wrap_exceptions @memoize_when_activated def _proc_basic_info(self): ret = cext.proc_basic_info(self.pid, self._procfs_path) assert len(ret) == len(proc_info_map) return ret + @wrap_exceptions @memoize_when_activated def _proc_cred(self): - @wrap_exceptions - def proc_cred(self): - return cext.proc_cred(self.pid, self._procfs_path) - return proc_cred(self) + return cext.proc_cred(self.pid, self._procfs_path) @wrap_exceptions def name(self): @@ -512,14 +518,11 @@ class Process(object): try: return os.readlink( '%s/%d/path/%d' % (procfs_path, self.pid, x)) - except OSError as err: - if err.errno == errno.ENOENT: - hit_enoent = True - continue - raise + except FileNotFoundError: + hit_enoent = True + continue if hit_enoent: - # raise NSP if the process disappeared on us - os.stat('%s/%s' % (procfs_path, self.pid)) + self._assert_alive() @wrap_exceptions def cwd(self): @@ -530,11 +533,9 @@ class Process(object): procfs_path = self._procfs_path try: return os.readlink("%s/%s/path/cwd" % (procfs_path, self.pid)) - except OSError as err: - if err.errno == errno.ENOENT: - os.stat("%s/%s" % (procfs_path, self.pid)) # raise NSP or AD - return None - raise + except FileNotFoundError: + os.stat("%s/%s" % (procfs_path, self.pid)) # raise NSP or AD + return None @wrap_exceptions def memory_info(self): @@ -581,8 +582,7 @@ class Process(object): nt = _common.pthread(tid, utime, stime) ret.append(nt) if hit_enoent: - # raise NSP if the process disappeared on us - os.stat('%s/%s' % (procfs_path, self.pid)) + self._assert_alive() return ret @wrap_exceptions @@ -596,18 +596,14 @@ class Process(object): if os.path.islink(path): try: file = os.readlink(path) - except OSError as err: - # ENOENT == file which is gone in the meantime - if err.errno == errno.ENOENT: - hit_enoent = True - continue - raise + except FileNotFoundError: + hit_enoent = True + continue else: if isfile_strict(file): retlist.append(_common.popenfile(file, int(fd))) if hit_enoent: - # raise NSP if the process disappeared on us - os.stat('%s/%s' % (procfs_path, self.pid)) + self._assert_alive() return retlist def _get_unix_sockets(self, pid): @@ -707,8 +703,7 @@ class Process(object): raise retlist.append((addr, perm, name, rss, anon, locked)) if hit_enoent: - # raise NSP if the process disappeared on us - os.stat('%s/%s' % (procfs_path, self.pid)) + self._assert_alive() return retlist @wrap_exceptions diff --git a/server/www/packages/packages-linux/x64/psutil/_psutil_linux.cpython-37m-x86_64-linux-gnu.so b/server/www/packages/packages-linux/x64/psutil/_psutil_linux.cpython-37m-x86_64-linux-gnu.so index 26278e2..3ff94fa 100755 Binary files a/server/www/packages/packages-linux/x64/psutil/_psutil_linux.cpython-37m-x86_64-linux-gnu.so and b/server/www/packages/packages-linux/x64/psutil/_psutil_linux.cpython-37m-x86_64-linux-gnu.so differ diff --git a/server/www/packages/packages-linux/x64/psutil/_psutil_posix.cpython-37m-x86_64-linux-gnu.so b/server/www/packages/packages-linux/x64/psutil/_psutil_posix.cpython-37m-x86_64-linux-gnu.so index 8916dd8..6f33c93 100755 Binary files a/server/www/packages/packages-linux/x64/psutil/_psutil_posix.cpython-37m-x86_64-linux-gnu.so and b/server/www/packages/packages-linux/x64/psutil/_psutil_posix.cpython-37m-x86_64-linux-gnu.so differ diff --git a/server/www/packages/packages-linux/x64/psutil/_pswindows.py b/server/www/packages/packages-linux/x64/psutil/_pswindows.py index b938d42..636b0af 100644 --- a/server/www/packages/packages-linux/x64/psutil/_pswindows.py +++ b/server/www/packages/packages-linux/x64/psutil/_pswindows.py @@ -27,29 +27,25 @@ except ImportError as err: # but if we get here it means this this was a wheel (or exe). msg = "this Windows version is too old (< Windows Vista); " msg += "psutil 3.4.2 is the latest version which supports Windows " - msg += "2000, XP and 2003 server; it may be possible that psutil " - msg += "will work if compiled from sources though" + msg += "2000, XP and 2003 server" raise RuntimeError(msg) else: raise from ._common import conn_tmap +from ._common import conn_to_ntuple from ._common import ENCODING from ._common import ENCODING_ERRS from ._common import isfile_strict +from ._common import memoize from ._common import memoize_when_activated from ._common import parse_environ_block -from ._common import sockfam_to_enum -from ._common import socktype_to_enum from ._common import usage_percent from ._compat import long from ._compat import lru_cache from ._compat import PY3 from ._compat import unicode from ._compat import xrange -from ._exceptions import AccessDenied -from ._exceptions import NoSuchProcess -from ._exceptions import TimeoutExpired from ._psutil_windows import ABOVE_NORMAL_PRIORITY_CLASS from ._psutil_windows import BELOW_NORMAL_PRIORITY_CLASS from ._psutil_windows import HIGH_PRIORITY_CLASS @@ -66,11 +62,14 @@ else: # http://msdn.microsoft.com/en-us/library/ms686219(v=vs.85).aspx __extra__all__ = [ "win_service_iter", "win_service_get", + # Process priority "ABOVE_NORMAL_PRIORITY_CLASS", "BELOW_NORMAL_PRIORITY_CLASS", - "HIGH_PRIORITY_CLASS", "IDLE_PRIORITY_CLASS", - "NORMAL_PRIORITY_CLASS", "REALTIME_PRIORITY_CLASS", - "CONN_DELETE_TCB", - "AF_LINK", + "HIGH_PRIORITY_CLASS", "IDLE_PRIORITY_CLASS", "NORMAL_PRIORITY_CLASS", + "REALTIME_PRIORITY_CLASS", + # IO priority + "IOPRIO_VERYLOW", "IOPRIO_LOW", "IOPRIO_NORMAL", "IOPRIO_HIGH", + # others + "CONN_DELETE_TCB", "AF_LINK", ] @@ -79,11 +78,9 @@ __extra__all__ = [ # ===================================================================== CONN_DELETE_TCB = "DELETE_TCB" -ACCESS_DENIED_ERRSET = frozenset([errno.EPERM, errno.EACCES, - cext.ERROR_ACCESS_DENIED]) -NO_SUCH_SERVICE_ERRSET = frozenset([cext.ERROR_INVALID_NAME, - cext.ERROR_SERVICE_DOES_NOT_EXIST]) HAS_PROC_IO_PRIORITY = hasattr(cext, "proc_io_priority_get") +HAS_GETLOADAVG = hasattr(cext, "getloadavg") +ERROR_PARTIAL_COPY = 299 if enum is None: @@ -119,6 +116,19 @@ if enum is not None: globals().update(Priority.__members__) +if enum is None: + IOPRIO_VERYLOW = 0 + IOPRIO_LOW = 1 + IOPRIO_NORMAL = 2 + IOPRIO_HIGH = 3 +else: + class IOPriority(enum.IntEnum): + IOPRIO_VERYLOW = 0 + IOPRIO_LOW = 1 + IOPRIO_NORMAL = 2 + IOPRIO_HIGH = 3 + globals().update(IOPriority.__members__) + pinfo_map = dict( num_handles=0, ctx_switches=1, @@ -144,6 +154,35 @@ pinfo_map = dict( mem_private=21, ) +# These objects get set on "import psutil" from the __init__.py +# file, see: https://github.com/giampaolo/psutil/issues/1402 +NoSuchProcess = None +ZombieProcess = None +AccessDenied = None +TimeoutExpired = None + +# More values at: https://stackoverflow.com/a/20804735/376587 +WIN_10 = (10, 0) +WIN_8 = (6, 2) +WIN_7 = (6, 1) +WIN_SERVER_2008 = (6, 0) +WIN_VISTA = (6, 0) +WIN_SERVER_2003 = (5, 2) +WIN_XP = (5, 1) + + +@lru_cache() +def get_winver(): + """Usage: + >>> if get_winver() <= WIN_VISTA: + ... ... + """ + wv = sys.getwindowsversion() + return (wv.major, wv.minor) + + +IS_WIN_XP = get_winver() < WIN_VISTA + # ===================================================================== # --- named tuples @@ -204,6 +243,11 @@ def py2_strencode(s): return s.encode(ENCODING, ENCODING_ERRS) +@memoize +def getpagesize(): + return cext.getpagesize() + + # ===================================================================== # --- memory # ===================================================================== @@ -310,6 +354,23 @@ def cpu_freq(): return [_common.scpufreq(float(curr), min_, float(max_))] +if HAS_GETLOADAVG: + _loadavg_inititialized = False + + def getloadavg(): + """Return the number of processes in the system run queue averaged + over the last 1, 5, and 15 minutes respectively as a tuple""" + global _loadavg_inititialized + + if not _loadavg_inititialized: + cext.init_loadavg_counter() + _loadavg_inititialized = True + + # Drop to 2 decimal points which is what Linux does + raw_loads = cext.getloadavg() + return tuple([round(load, 2) for load in raw_loads]) + + # ===================================================================== # --- network # ===================================================================== @@ -327,17 +388,8 @@ def net_connections(kind, _pid=-1): ret = set() for item in rawlist: fd, fam, type, laddr, raddr, status, pid = item - if laddr: - laddr = _common.addr(*laddr) - if raddr: - raddr = _common.addr(*raddr) - status = TCP_STATUSES[status] - fam = sockfam_to_enum(fam) - type = socktype_to_enum(type) - if _pid == -1: - nt = _common.sconn(fd, fam, type, laddr, raddr, status, pid) - else: - nt = _common.pconn(fd, fam, type, laddr, raddr, status) + nt = conn_to_ntuple(fd, fam, type, laddr, raddr, status, TCP_STATUSES, + pid=pid if _pid == -1 else None) ret.add(nt) return list(ret) @@ -502,14 +554,14 @@ class WindowsService(object): """ try: yield - except WindowsError as err: - if err.errno in ACCESS_DENIED_ERRSET: + except OSError as err: + if is_permission_err(err): raise AccessDenied( pid=None, name=self._name, msg="service %r is not querable (not enough privileges)" % self._name) - elif err.errno in NO_SUCH_SERVICE_ERRSET or \ - err.winerror in NO_SUCH_SERVICE_ERRSET: + elif err.winerror in (cext.ERROR_INVALID_NAME, + cext.ERROR_SERVICE_DOES_NOT_EXIST): raise NoSuchProcess( pid=None, name=self._name, msg="service %r does not exist)" % self._name) @@ -626,27 +678,68 @@ pid_exists = cext.pid_exists ppid_map = cext.ppid_map # used internally by Process.children() +def is_permission_err(exc): + """Return True if this is a permission error.""" + assert isinstance(exc, OSError), exc + # On Python 2 OSError doesn't always have 'winerror'. Sometimes + # it does, in which case the original exception was WindowsError + # (which is a subclass of OSError). + return exc.errno in (errno.EPERM, errno.EACCES) or \ + getattr(exc, "winerror", -1) in (cext.ERROR_ACCESS_DENIED, + cext.ERROR_PRIVILEGE_NOT_HELD) + + +def convert_oserror(exc, pid=None, name=None): + """Convert OSError into NoSuchProcess or AccessDenied.""" + assert isinstance(exc, OSError), exc + if is_permission_err(exc): + return AccessDenied(pid=pid, name=name) + if exc.errno == errno.ESRCH: + return NoSuchProcess(pid=pid, name=name) + raise exc + + def wrap_exceptions(fun): - """Decorator which translates bare OSError and WindowsError - exceptions into NoSuchProcess and AccessDenied. - """ + """Decorator which converts OSError into NoSuchProcess or AccessDenied.""" @functools.wraps(fun) def wrapper(self, *args, **kwargs): try: return fun(self, *args, **kwargs) except OSError as err: - if err.errno in ACCESS_DENIED_ERRSET: - raise AccessDenied(self.pid, self._name) - if err.errno == errno.ESRCH: - raise NoSuchProcess(self.pid, self._name) - raise + raise convert_oserror(err, pid=self.pid, name=self._name) + return wrapper + + +def retry_error_partial_copy(fun): + """Workaround for https://github.com/giampaolo/psutil/issues/875. + See: https://stackoverflow.com/questions/4457745#4457745 + """ + @functools.wraps(fun) + def wrapper(self, *args, **kwargs): + delay = 0.0001 + times = 33 + for x in range(times): # retries for roughly 1 second + try: + return fun(self, *args, **kwargs) + except WindowsError as _: + err = _ + if err.winerror == ERROR_PARTIAL_COPY: + time.sleep(delay) + delay = min(delay * 2, 0.04) + continue + else: + raise + else: + msg = "%s retried %s times, converted to AccessDenied as it's " \ + "still returning %r" % (fun, times, err) + raise AccessDenied(pid=self.pid, name=self._name, msg=msg) return wrapper class Process(object): """Wrapper class around underlying C implementation.""" - __slots__ = ["pid", "_name", "_ppid"] + __slots__ = ["pid", "_name", "_ppid", "_cache"] def __init__(self, pid): self.pid = pid @@ -656,11 +749,12 @@ class Process(object): # --- oneshot() stuff def oneshot_enter(self): - self.oneshot_info.cache_activate() + self.oneshot_info.cache_activate(self) def oneshot_exit(self): - self.oneshot_info.cache_deactivate() + self.oneshot_info.cache_deactivate(self) + @wrap_exceptions @memoize_when_activated def oneshot_info(self): """Return multiple information about this process as a @@ -691,25 +785,41 @@ class Process(object): @wrap_exceptions def exe(self): - # Note: os.path.exists(path) may return False even if the file - # is there, see: - # http://stackoverflow.com/questions/3112546/os-path-exists-lies - - # see https://github.com/giampaolo/psutil/issues/414 - # see https://github.com/giampaolo/psutil/issues/528 - if self.pid in (0, 4): - raise AccessDenied(self.pid, self._name) - return py2_strencode(convert_dos_path(cext.proc_exe(self.pid))) + # Dual implementation, see: + # https://github.com/giampaolo/psutil/pull/1413 + if not IS_WIN_XP: + exe = cext.proc_exe(self.pid) + else: + if self.pid in (0, 4): + # https://github.com/giampaolo/psutil/issues/414 + # https://github.com/giampaolo/psutil/issues/528 + raise AccessDenied(self.pid, self._name) + exe = cext.proc_exe(self.pid) + exe = convert_dos_path(exe) + return py2_strencode(exe) @wrap_exceptions + @retry_error_partial_copy def cmdline(self): - ret = cext.proc_cmdline(self.pid) + if cext.WINVER >= cext.WINDOWS_8_1: + # PEB method detects cmdline changes but requires more + # privileges: https://github.com/giampaolo/psutil/pull/1398 + try: + ret = cext.proc_cmdline(self.pid, use_peb=True) + except OSError as err: + if is_permission_err(err): + ret = cext.proc_cmdline(self.pid, use_peb=False) + else: + raise + else: + ret = cext.proc_cmdline(self.pid, use_peb=True) if PY3: return ret else: return [py2_strencode(s) for s in ret] @wrap_exceptions + @retry_error_partial_copy def environ(self): ustr = cext.proc_environ(self.pid) if ustr and not PY3: @@ -726,7 +836,7 @@ class Process(object): try: return cext.proc_memory_info(self.pid) except OSError as err: - if err.errno in ACCESS_DENIED_ERRSET: + if is_permission_err(err): # TODO: the C ext can probably be refactored in order # to get this from cext.proc_info() info = self.oneshot_info() @@ -758,6 +868,7 @@ class Process(object): def memory_full_info(self): basic_mem = self.memory_info() uss = cext.proc_memory_uss(self.pid) + uss *= getpagesize() return pfullmem(*basic_mem + (uss, )) def memory_maps(self): @@ -766,11 +877,7 @@ class Process(object): except OSError as err: # XXX - can't use wrap_exceptions decorator as we're # returning a generator; probably needs refactoring. - if err.errno in ACCESS_DENIED_ERRSET: - raise AccessDenied(self.pid, self._name) - if err.errno == errno.ESRCH: - raise NoSuchProcess(self.pid, self._name) - raise + raise convert_oserror(err, self.pid, self._name) else: for addr, perm, path, rss in raw: path = convert_dos_path(path) @@ -846,7 +953,7 @@ class Process(object): try: return cext.proc_create_time(self.pid) except OSError as err: - if err.errno in ACCESS_DENIED_ERRSET: + if is_permission_err(err): return self.oneshot_info()[pinfo_map['create_time']] raise @@ -868,24 +975,24 @@ class Process(object): try: user, system = cext.proc_cpu_times(self.pid) except OSError as err: - if err.errno in ACCESS_DENIED_ERRSET: - info = self.oneshot_info() - user = info[pinfo_map['user_time']] - system = info[pinfo_map['kernel_time']] - else: + if not is_permission_err(err): raise + info = self.oneshot_info() + user = info[pinfo_map['user_time']] + system = info[pinfo_map['kernel_time']] # Children user/system times are not retrievable (set to 0). return _common.pcputimes(user, system, 0.0, 0.0) @wrap_exceptions def suspend(self): - return cext.proc_suspend(self.pid) + cext.proc_suspend_or_resume(self.pid, True) @wrap_exceptions def resume(self): - return cext.proc_resume(self.pid) + cext.proc_suspend_or_resume(self.pid, False) @wrap_exceptions + @retry_error_partial_copy def cwd(self): if self.pid in (0, 4): raise AccessDenied(self.pid, self._name) @@ -932,35 +1039,36 @@ class Process(object): if HAS_PROC_IO_PRIORITY: @wrap_exceptions def ionice_get(self): - return cext.proc_io_priority_get(self.pid) + ret = cext.proc_io_priority_get(self.pid) + if enum is not None: + ret = IOPriority(ret) + return ret @wrap_exceptions - def ionice_set(self, value, _): - if _: - raise TypeError("set_proc_ionice() on Windows takes only " - "1 argument (2 given)") - if value not in (2, 1, 0): - raise ValueError("value must be 2 (normal), 1 (low) or 0 " - "(very low); got %r" % value) - return cext.proc_io_priority_set(self.pid, value) + def ionice_set(self, ioclass, value): + if value: + raise TypeError("value argument not accepted on Windows") + if ioclass not in (IOPRIO_VERYLOW, IOPRIO_LOW, IOPRIO_NORMAL, + IOPRIO_HIGH): + raise ValueError("%s is not a valid priority" % ioclass) + cext.proc_io_priority_set(self.pid, ioclass) @wrap_exceptions def io_counters(self): try: ret = cext.proc_io_counters(self.pid) except OSError as err: - if err.errno in ACCESS_DENIED_ERRSET: - info = self.oneshot_info() - ret = ( - info[pinfo_map['io_rcount']], - info[pinfo_map['io_wcount']], - info[pinfo_map['io_rbytes']], - info[pinfo_map['io_wbytes']], - info[pinfo_map['io_count_others']], - info[pinfo_map['io_bytes_others']], - ) - else: + if not is_permission_err(err): raise + info = self.oneshot_info() + ret = ( + info[pinfo_map['io_rcount']], + info[pinfo_map['io_wcount']], + info[pinfo_map['io_rbytes']], + info[pinfo_map['io_wbytes']], + info[pinfo_map['io_count_others']], + info[pinfo_map['io_bytes_others']], + ) return pio(*ret) @wrap_exceptions @@ -1008,7 +1116,7 @@ class Process(object): try: return cext.proc_num_handles(self.pid) except OSError as err: - if err.errno in ACCESS_DENIED_ERRSET: + if is_permission_err(err): return self.oneshot_info()[pinfo_map['num_handles']] raise diff --git a/server/www/packages/packages-linux/x64/pyasn1/__init__.py b/server/www/packages/packages-linux/x64/pyasn1/__init__.py index e2e4c5c..5a56a70 100644 --- a/server/www/packages/packages-linux/x64/pyasn1/__init__.py +++ b/server/www/packages/packages-linux/x64/pyasn1/__init__.py @@ -1,7 +1,7 @@ import sys # https://www.python.org/dev/peps/pep-0396/ -__version__ = '0.4.4' +__version__ = '0.4.8' if sys.version_info[:2] < (2, 4): raise RuntimeError('PyASN1 requires Python 2.4 or later') diff --git a/server/www/packages/packages-linux/x64/pyasn1/codec/ber/decoder.py b/server/www/packages/packages-linux/x64/pyasn1/codec/ber/decoder.py index a27b3e0..5ff485f 100644 --- a/server/www/packages/packages-linux/x64/pyasn1/codec/ber/decoder.py +++ b/server/www/packages/packages-linux/x64/pyasn1/codec/ber/decoder.py @@ -1,7 +1,7 @@ # # This file is part of pyasn1 software. # -# Copyright (c) 2005-2018, Ilya Etingof +# Copyright (c) 2005-2019, Ilya Etingof # License: http://snmplabs.com/pyasn1/license.html # from pyasn1 import debug @@ -18,6 +18,8 @@ from pyasn1.type import useful __all__ = ['decode'] +LOG = debug.registerLoggee(__name__, flags=debug.DEBUG_DECODER) + noValue = base.noValue @@ -70,6 +72,10 @@ class ExplicitTagDecoder(AbstractSimpleDecoder): value, _ = decodeFun(head, asn1Spec, tagSet, length, **options) + if LOG: + LOG('explicit tag container carries %d octets of trailing payload ' + '(will be lost!): %s' % (len(_), debug.hexdump(_))) + return value, tail def indefLenValueDecoder(self, substrate, asn1Spec, @@ -120,7 +126,8 @@ class BooleanDecoder(IntegerDecoder): protoComponent = univ.Boolean(0) def _createComponent(self, asn1Spec, tagSet, value, **options): - return IntegerDecoder._createComponent(self, asn1Spec, tagSet, value and 1 or 0, **options) + return IntegerDecoder._createComponent( + self, asn1Spec, tagSet, value and 1 or 0, **options) class BitStringDecoder(AbstractSimpleDecoder): @@ -134,8 +141,8 @@ class BitStringDecoder(AbstractSimpleDecoder): head, tail = substrate[:length], substrate[length:] if substrateFun: - return substrateFun(self._createComponent(asn1Spec, tagSet, noValue, **options), - substrate, length) + return substrateFun(self._createComponent( + asn1Spec, tagSet, noValue, **options), substrate, length) if not head: raise error.PyAsn1Error('Empty BIT STRING substrate') @@ -148,12 +155,17 @@ class BitStringDecoder(AbstractSimpleDecoder): 'Trailing bits overflow %s' % trailingBits ) - value = self.protoComponent.fromOctetString(head[1:], internalFormat=True, padding=trailingBits) + value = self.protoComponent.fromOctetString( + head[1:], internalFormat=True, padding=trailingBits) return self._createComponent(asn1Spec, tagSet, value, **options), tail if not self.supportConstructedForm: - raise error.PyAsn1Error('Constructed encoding form prohibited at %s' % self.__class__.__name__) + raise error.PyAsn1Error('Constructed encoding form prohibited ' + 'at %s' % self.__class__.__name__) + + if LOG: + LOG('assembling constructed serialization') # All inner fragments are of the same type, treat them as octet string substrateFun = self.substrateCollector @@ -234,6 +246,9 @@ class OctetStringDecoder(AbstractSimpleDecoder): if not self.supportConstructedForm: raise error.PyAsn1Error('Constructed encoding form prohibited at %s' % self.__class__.__name__) + if LOG: + LOG('assembling constructed serialization') + # All inner fragments are of the same type, treat them as octet string substrateFun = self.substrateCollector @@ -267,7 +282,9 @@ class OctetStringDecoder(AbstractSimpleDecoder): allowEoo=True, **options) if component is eoo.endOfOctets: break + header += component + else: raise error.SubstrateUnderrunError( 'No EOO seen before substrate ends' @@ -374,59 +391,90 @@ class RealDecoder(AbstractSimpleDecoder): if fo & 0x80: # binary encoding if not head: raise error.PyAsn1Error("Incomplete floating-point value") + + if LOG: + LOG('decoding binary encoded REAL') + n = (fo & 0x03) + 1 + if n == 4: n = oct2int(head[0]) head = head[1:] + eo, head = head[:n], head[n:] + if not eo or not head: raise error.PyAsn1Error('Real exponent screwed') + e = oct2int(eo[0]) & 0x80 and -1 or 0 + while eo: # exponent e <<= 8 e |= oct2int(eo[0]) eo = eo[1:] + b = fo >> 4 & 0x03 # base bits + if b > 2: raise error.PyAsn1Error('Illegal Real base') + if b == 1: # encbase = 8 e *= 3 + elif b == 2: # encbase = 16 e *= 4 p = 0 + while head: # value p <<= 8 p |= oct2int(head[0]) head = head[1:] + if fo & 0x40: # sign bit p = -p + sf = fo >> 2 & 0x03 # scale bits p *= 2 ** sf value = (p, 2, e) + elif fo & 0x40: # infinite value + if LOG: + LOG('decoding infinite REAL') + value = fo & 0x01 and '-inf' or 'inf' + elif fo & 0xc0 == 0: # character encoding if not head: raise error.PyAsn1Error("Incomplete floating-point value") + + if LOG: + LOG('decoding character encoded REAL') + try: if fo & 0x3 == 0x1: # NR1 value = (int(head), 10, 0) + elif fo & 0x3 == 0x2: # NR2 value = float(head) + elif fo & 0x3 == 0x3: # NR3 value = float(head) + else: raise error.SubstrateUnderrunError( 'Unknown NR (tag %s)' % fo ) + except ValueError: raise error.SubstrateUnderrunError( 'Bad character Real syntax' ) + else: raise error.SubstrateUnderrunError( 'Unknown encoding (tag %s)' % fo ) + return self._createComponent(asn1Spec, tagSet, value, **options), tail @@ -447,10 +495,12 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): def _decodeComponents(self, substrate, tagSet=None, decodeFun=None, **options): components = [] componentTypes = set() + while substrate: component, substrate = decodeFun(substrate, **options) if component is eoo.endOfOctets: break + components.append(component) componentTypes.add(component.tagSet) @@ -460,6 +510,7 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): # * otherwise -> likely SEQUENCE OF/SET OF if len(componentTypes) > 1: protoComponent = self.protoRecordComponent + else: protoComponent = self.protoSequenceComponent @@ -469,6 +520,10 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): tagSet=tag.TagSet(protoComponent.tagSet.baseTag, *tagSet.superTags) ) + if LOG: + LOG('guessed %r container type (pass `asn1Spec` to guide the ' + 'decoder)' % asn1Object) + for idx, component in enumerate(components): asn1Object.setComponentByPosition( idx, component, @@ -490,8 +545,10 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): if substrateFun is not None: if asn1Spec is not None: asn1Object = asn1Spec.clone() + elif self.protoComponent is not None: asn1Object = self.protoComponent.clone(tagSet=tagSet) + else: asn1Object = self.protoRecordComponent, self.protoSequenceComponent @@ -501,11 +558,16 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): asn1Object, trailing = self._decodeComponents( head, tagSet=tagSet, decodeFun=decodeFun, **options ) + if trailing: - raise error.PyAsn1Error('Unused trailing %d octets encountered' % len(trailing)) + if LOG: + LOG('Unused trailing %d octets encountered: %s' % ( + len(trailing), debug.hexdump(trailing))) + return asn1Object, tail asn1Object = asn1Spec.clone() + asn1Object.clear() if asn1Spec.typeId in (univ.Sequence.typeId, univ.Set.typeId): @@ -514,21 +576,31 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): isSetType = asn1Spec.typeId == univ.Set.typeId isDeterministic = not isSetType and not namedTypes.hasOptionalOrDefault + if LOG: + LOG('decoding %sdeterministic %s type %r chosen by type ID' % ( + not isDeterministic and 'non-' or '', isSetType and 'SET' or '', + asn1Spec)) + seenIndices = set() idx = 0 while head: if not namedTypes: componentType = None + elif isSetType: componentType = namedTypes.tagMapUnique + else: try: if isDeterministic: componentType = namedTypes[idx].asn1Object + elif namedTypes[idx].isOptional or namedTypes[idx].isDefaulted: componentType = namedTypes.getTagMapNearPosition(idx) + else: componentType = namedTypes[idx].asn1Object + except IndexError: raise error.PyAsn1Error( 'Excessive components decoded at %r' % (asn1Spec,) @@ -539,6 +611,7 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): if not isDeterministic and namedTypes: if isSetType: idx = namedTypes.getPositionByType(component.effectiveTagSet) + elif namedTypes[idx].isOptional or namedTypes[idx].isDefaulted: idx = namedTypes.getPositionNearType(component.effectiveTagSet, idx) @@ -551,14 +624,25 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): seenIndices.add(idx) idx += 1 + if LOG: + LOG('seen component indices %s' % seenIndices) + if namedTypes: if not namedTypes.requiredComponents.issubset(seenIndices): - raise error.PyAsn1Error('ASN.1 object %s has uninitialized components' % asn1Object.__class__.__name__) + raise error.PyAsn1Error( + 'ASN.1 object %s has uninitialized ' + 'components' % asn1Object.__class__.__name__) if namedTypes.hasOpenTypes: openTypes = options.get('openTypes', {}) + if LOG: + LOG('user-specified open types map:') + + for k, v in openTypes.items(): + LOG('%s -> %r' % (k, v)) + if openTypes or options.get('decodeOpenTypes', False): for idx, namedType in enumerate(namedTypes.namedTypes): @@ -577,27 +661,67 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): except KeyError: + if LOG: + LOG('default open types map of component ' + '"%s.%s" governed by component "%s.%s"' + ':' % (asn1Object.__class__.__name__, + namedType.name, + asn1Object.__class__.__name__, + namedType.openType.name)) + + for k, v in namedType.openType.items(): + LOG('%s -> %r' % (k, v)) + try: openType = namedType.openType[governingValue] except KeyError: + if LOG: + LOG('failed to resolve open type by governing ' + 'value %r' % (governingValue,)) continue - component, rest = decodeFun( - asn1Object.getComponentByPosition(idx).asOctets(), - asn1Spec=openType - ) + if LOG: + LOG('resolved open type %r by governing ' + 'value %r' % (openType, governingValue)) - asn1Object.setComponentByPosition(idx, component) + containerValue = asn1Object.getComponentByPosition(idx) + + if containerValue.typeId in ( + univ.SetOf.typeId, univ.SequenceOf.typeId): + + for pos, containerElement in enumerate( + containerValue): + + component, rest = decodeFun( + containerValue[pos].asOctets(), + asn1Spec=openType, **options + ) + + containerValue[pos] = component + + else: + component, rest = decodeFun( + asn1Object.getComponentByPosition(idx).asOctets(), + asn1Spec=openType, **options + ) + + asn1Object.setComponentByPosition(idx, component) else: - asn1Object.verifySizeSpec() + inconsistency = asn1Object.isInconsistent + if inconsistency: + raise inconsistency else: asn1Object = asn1Spec.clone() + asn1Object.clear() componentType = asn1Spec.componentType + if LOG: + LOG('decoding type %r chosen by given `asn1Spec`' % componentType) + idx = 0 while head: @@ -607,6 +731,7 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): verifyConstraints=False, matchTags=False, matchConstraints=False ) + idx += 1 return asn1Object, tail @@ -621,8 +746,10 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): if substrateFun is not None: if asn1Spec is not None: asn1Object = asn1Spec.clone() + elif self.protoComponent is not None: asn1Object = self.protoComponent.clone(tagSet=tagSet) + else: asn1Object = self.protoRecordComponent, self.protoSequenceComponent @@ -630,10 +757,12 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): if asn1Spec is None: return self._decodeComponents( - substrate, tagSet=tagSet, decodeFun=decodeFun, allowEoo=True, **options + substrate, tagSet=tagSet, decodeFun=decodeFun, + **dict(options, allowEoo=True) ) asn1Object = asn1Spec.clone() + asn1Object.clear() if asn1Spec.typeId in (univ.Sequence.typeId, univ.Set.typeId): @@ -642,21 +771,31 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): isSetType = asn1Object.typeId == univ.Set.typeId isDeterministic = not isSetType and not namedTypes.hasOptionalOrDefault + if LOG: + LOG('decoding %sdeterministic %s type %r chosen by type ID' % ( + not isDeterministic and 'non-' or '', isSetType and 'SET' or '', + asn1Spec)) + seenIndices = set() idx = 0 while substrate: if len(namedTypes) <= idx: asn1Spec = None + elif isSetType: asn1Spec = namedTypes.tagMapUnique + else: try: if isDeterministic: asn1Spec = namedTypes[idx].asn1Object + elif namedTypes[idx].isOptional or namedTypes[idx].isDefaulted: asn1Spec = namedTypes.getTagMapNearPosition(idx) + else: asn1Spec = namedTypes[idx].asn1Object + except IndexError: raise error.PyAsn1Error( 'Excessive components decoded at %r' % (asn1Object,) @@ -686,13 +825,22 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): 'No EOO seen before substrate ends' ) + if LOG: + LOG('seen component indices %s' % seenIndices) + if namedTypes: if not namedTypes.requiredComponents.issubset(seenIndices): raise error.PyAsn1Error('ASN.1 object %s has uninitialized components' % asn1Object.__class__.__name__) - if namedTypes.hasOpenTypes: + if namedTypes.hasOpenTypes: - openTypes = options.get('openTypes', None) + openTypes = options.get('openTypes', {}) + + if LOG: + LOG('user-specified open types map:') + + for k, v in openTypes.items(): + LOG('%s -> %r' % (k, v)) if openTypes or options.get('decodeOpenTypes', False): @@ -712,28 +860,68 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): except KeyError: + if LOG: + LOG('default open types map of component ' + '"%s.%s" governed by component "%s.%s"' + ':' % (asn1Object.__class__.__name__, + namedType.name, + asn1Object.__class__.__name__, + namedType.openType.name)) + + for k, v in namedType.openType.items(): + LOG('%s -> %r' % (k, v)) + try: openType = namedType.openType[governingValue] except KeyError: + if LOG: + LOG('failed to resolve open type by governing ' + 'value %r' % (governingValue,)) continue - component, rest = decodeFun( - asn1Object.getComponentByPosition(idx).asOctets(), - asn1Spec=openType, allowEoo=True - ) + if LOG: + LOG('resolved open type %r by governing ' + 'value %r' % (openType, governingValue)) - if component is not eoo.endOfOctets: - asn1Object.setComponentByPosition(idx, component) + containerValue = asn1Object.getComponentByPosition(idx) + + if containerValue.typeId in ( + univ.SetOf.typeId, univ.SequenceOf.typeId): + + for pos, containerElement in enumerate( + containerValue): + + component, rest = decodeFun( + containerValue[pos].asOctets(), + asn1Spec=openType, **dict(options, allowEoo=True) + ) + + containerValue[pos] = component + + else: + component, rest = decodeFun( + asn1Object.getComponentByPosition(idx).asOctets(), + asn1Spec=openType, **dict(options, allowEoo=True) + ) + + if component is not eoo.endOfOctets: + asn1Object.setComponentByPosition(idx, component) else: - asn1Object.verifySizeSpec() + inconsistency = asn1Object.isInconsistent + if inconsistency: + raise inconsistency else: asn1Object = asn1Spec.clone() + asn1Object.clear() componentType = asn1Spec.componentType + if LOG: + LOG('decoding type %r chosen by given `asn1Spec`' % componentType) + idx = 0 while substrate: @@ -747,7 +935,9 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): verifyConstraints=False, matchTags=False, matchConstraints=False ) + idx += 1 + else: raise error.SubstrateUnderrunError( 'No EOO seen before substrate ends' @@ -794,18 +984,25 @@ class ChoiceDecoder(AbstractConstructedDecoder): if asn1Spec is None: asn1Object = self.protoComponent.clone(tagSet=tagSet) + else: asn1Object = asn1Spec.clone() if substrateFun: return substrateFun(asn1Object, substrate, length) - if asn1Object.tagSet == tagSet: # explicitly tagged Choice + if asn1Object.tagSet == tagSet: + if LOG: + LOG('decoding %s as explicitly tagged CHOICE' % (tagSet,)) + component, head = decodeFun( head, asn1Object.componentTagMap, **options ) else: + if LOG: + LOG('decoding %s as untagged CHOICE' % (tagSet,)) + component, head = decodeFun( head, asn1Object.componentTagMap, tagSet, length, state, **options @@ -813,6 +1010,9 @@ class ChoiceDecoder(AbstractConstructedDecoder): effectiveTagSet = component.effectiveTagSet + if LOG: + LOG('decoded component %s, effective tag set %s' % (component, effectiveTagSet)) + asn1Object.setComponentByType( effectiveTagSet, component, verifyConstraints=False, @@ -834,18 +1034,26 @@ class ChoiceDecoder(AbstractConstructedDecoder): if substrateFun: return substrateFun(asn1Object, substrate, length) - if asn1Object.tagSet == tagSet: # explicitly tagged Choice + if asn1Object.tagSet == tagSet: + if LOG: + LOG('decoding %s as explicitly tagged CHOICE' % (tagSet,)) + component, substrate = decodeFun( substrate, asn1Object.componentType.tagMapUnique, **options ) + # eat up EOO marker eooMarker, substrate = decodeFun( substrate, allowEoo=True, **options ) + if eooMarker is not eoo.endOfOctets: raise error.PyAsn1Error('No EOO seen before substrate ends') else: + if LOG: + LOG('decoding %s as untagged CHOICE' % (tagSet,)) + component, substrate = decodeFun( substrate, asn1Object.componentType.tagMapUnique, tagSet, length, state, **options @@ -853,6 +1061,9 @@ class ChoiceDecoder(AbstractConstructedDecoder): effectiveTagSet = component.effectiveTagSet + if LOG: + LOG('decoded component %s, effective tag set %s' % (component, effectiveTagSet)) + asn1Object.setComponentByType( effectiveTagSet, component, verifyConstraints=False, @@ -870,13 +1081,25 @@ class AnyDecoder(AbstractSimpleDecoder): tagSet=None, length=None, state=None, decodeFun=None, substrateFun=None, **options): - if asn1Spec is None or asn1Spec is not None and tagSet != asn1Spec.tagSet: + if asn1Spec is None: + isUntagged = True + + elif asn1Spec.__class__ is tagmap.TagMap: + isUntagged = tagSet not in asn1Spec.tagMap + + else: + isUntagged = tagSet != asn1Spec.tagSet + + if isUntagged: fullSubstrate = options['fullSubstrate'] # untagged Any container, recover inner header substrate length += len(fullSubstrate) - len(substrate) substrate = fullSubstrate + if LOG: + LOG('decoding as untagged ANY, substrate %s' % debug.hexdump(substrate)) + if substrateFun: return substrateFun(self._createComponent(asn1Spec, tagSet, noValue, **options), substrate, length) @@ -889,15 +1112,31 @@ class AnyDecoder(AbstractSimpleDecoder): tagSet=None, length=None, state=None, decodeFun=None, substrateFun=None, **options): - if asn1Spec is not None and tagSet == asn1Spec.tagSet: + if asn1Spec is None: + isTagged = False + + elif asn1Spec.__class__ is tagmap.TagMap: + isTagged = tagSet in asn1Spec.tagMap + + else: + isTagged = tagSet == asn1Spec.tagSet + + if isTagged: # tagged Any type -- consume header substrate header = null + + if LOG: + LOG('decoding as tagged ANY') + else: fullSubstrate = options['fullSubstrate'] # untagged Any, recover header substrate header = fullSubstrate[:-len(substrate)] + if LOG: + LOG('decoding as untagged ANY, header substrate %s' % debug.hexdump(header)) + # Any components do not inherit initial tag asn1Spec = self.protoComponent @@ -905,6 +1144,9 @@ class AnyDecoder(AbstractSimpleDecoder): asn1Object = self._createComponent(asn1Spec, tagSet, noValue, **options) return substrateFun(asn1Object, header + substrate, length + len(header)) + if LOG: + LOG('assembling constructed serialization') + # All inner fragments are of the same type, treat them as octet string substrateFun = self.substrateCollector @@ -914,13 +1156,17 @@ class AnyDecoder(AbstractSimpleDecoder): allowEoo=True, **options) if component is eoo.endOfOctets: break + header += component + else: raise error.SubstrateUnderrunError( 'No EOO seen before substrate ends' ) + if substrateFun: return header, substrate + else: return self._createComponent(asn1Spec, tagSet, header, **options), substrate @@ -1045,7 +1291,7 @@ for typeDecoder in tagMap.values(): class Decoder(object): defaultErrorState = stErrorCondition - # defaultErrorState = stDumpRawValue + #defaultErrorState = stDumpRawValue defaultRawDecoder = AnyDecoder() supportIndefLength = True @@ -1063,21 +1309,16 @@ class Decoder(object): decodeFun=None, substrateFun=None, **options): - if debug.logger & debug.flagDecoder: - logger = debug.logger - else: - logger = None - - if logger: - logger('decoder called at scope %s with state %d, working with up to %d octets of substrate: %s' % (debug.scope, state, len(substrate), debug.hexdump(substrate))) + if LOG: + LOG('decoder called at scope %s with state %d, working with up to %d octets of substrate: %s' % (debug.scope, state, len(substrate), debug.hexdump(substrate))) allowEoo = options.pop('allowEoo', False) # Look for end-of-octets sentinel if allowEoo and self.supportIndefLength: if substrate[:2] == self.__eooSentinel: - if logger: - logger('end-of-octets sentinel found') + if LOG: + LOG('end-of-octets sentinel found') return eoo.endOfOctets, substrate[2:] value = noValue @@ -1090,26 +1331,32 @@ class Decoder(object): fullSubstrate = substrate while state is not stStop: + if state is stDecodeTag: if not substrate: raise error.SubstrateUnderrunError( 'Short octet stream on tag decoding' ) + # Decode tag isShortTag = True firstOctet = substrate[0] substrate = substrate[1:] + try: lastTag = tagCache[firstOctet] + except KeyError: integerTag = oct2int(firstOctet) tagClass = integerTag & 0xC0 tagFormat = integerTag & 0x20 tagId = integerTag & 0x1F + if tagId == 0x1F: isShortTag = False lengthOctetIdx = 0 tagId = 0 + try: while True: integerTag = oct2int(substrate[lengthOctetIdx]) @@ -1118,42 +1365,55 @@ class Decoder(object): tagId |= (integerTag & 0x7F) if not integerTag & 0x80: break + substrate = substrate[lengthOctetIdx:] + except IndexError: raise error.SubstrateUnderrunError( 'Short octet stream on long tag decoding' ) + lastTag = tag.Tag( tagClass=tagClass, tagFormat=tagFormat, tagId=tagId ) + if isShortTag: # cache short tags tagCache[firstOctet] = lastTag + if tagSet is None: if isShortTag: try: tagSet = tagSetCache[firstOctet] + except KeyError: # base tag not recovered tagSet = tag.TagSet((), lastTag) tagSetCache[firstOctet] = tagSet else: tagSet = tag.TagSet((), lastTag) + else: tagSet = lastTag + tagSet + state = stDecodeLength - if logger: - logger('tag decoded into %s, decoding length' % tagSet) + + if LOG: + LOG('tag decoded into %s, decoding length' % tagSet) + if state is stDecodeLength: # Decode length if not substrate: raise error.SubstrateUnderrunError( 'Short octet stream on length decoding' ) + firstOctet = oct2int(substrate[0]) + if firstOctet < 128: size = 1 length = firstOctet + elif firstOctet > 128: size = firstOctet & 0x7F # encoded in size bytes @@ -1164,28 +1424,36 @@ class Decoder(object): raise error.SubstrateUnderrunError( '%s<%s at %s' % (size, len(encodedLength), tagSet) ) + length = 0 for lengthOctet in encodedLength: length <<= 8 length |= lengthOctet size += 1 + else: size = 1 length = -1 substrate = substrate[size:] + if length == -1: if not self.supportIndefLength: raise error.PyAsn1Error('Indefinite length encoding not supported by this codec') + else: if len(substrate) < length: raise error.SubstrateUnderrunError('%d-octet short' % (length - len(substrate))) + state = stGetValueDecoder - if logger: - logger('value length decoded into %d, payload substrate is: %s' % (length, debug.hexdump(length == -1 and substrate or substrate[:length]))) + + if LOG: + LOG('value length decoded into %d, payload substrate is: %s' % (length, debug.hexdump(length == -1 and substrate or substrate[:length]))) + if state is stGetValueDecoder: if asn1Spec is None: state = stGetValueDecoderByTag + else: state = stGetValueDecoderByAsn1Spec # @@ -1207,41 +1475,55 @@ class Decoder(object): if state is stGetValueDecoderByTag: try: concreteDecoder = tagMap[tagSet] + except KeyError: concreteDecoder = None + if concreteDecoder: state = stDecodeValue + else: try: concreteDecoder = tagMap[tagSet[:1]] + except KeyError: concreteDecoder = None + if concreteDecoder: state = stDecodeValue else: state = stTryAsExplicitTag - if logger: - logger('codec %s chosen by a built-in type, decoding %s' % (concreteDecoder and concreteDecoder.__class__.__name__ or "", state is stDecodeValue and 'value' or 'as explicit tag')) + + if LOG: + LOG('codec %s chosen by a built-in type, decoding %s' % (concreteDecoder and concreteDecoder.__class__.__name__ or "", state is stDecodeValue and 'value' or 'as explicit tag')) debug.scope.push(concreteDecoder is None and '?' or concreteDecoder.protoComponent.__class__.__name__) + if state is stGetValueDecoderByAsn1Spec: + if asn1Spec.__class__ is tagmap.TagMap: try: chosenSpec = asn1Spec[tagSet] + except KeyError: chosenSpec = None - if logger: - logger('candidate ASN.1 spec is a map of:') + + if LOG: + LOG('candidate ASN.1 spec is a map of:') + for firstOctet, v in asn1Spec.presentTypes.items(): - logger(' %s -> %s' % (firstOctet, v.__class__.__name__)) + LOG(' %s -> %s' % (firstOctet, v.__class__.__name__)) + if asn1Spec.skipTypes: - logger('but neither of: ') + LOG('but neither of: ') for firstOctet, v in asn1Spec.skipTypes.items(): - logger(' %s -> %s' % (firstOctet, v.__class__.__name__)) - logger('new candidate ASN.1 spec is %s, chosen by %s' % (chosenSpec is None and '' or chosenSpec.prettyPrintType(), tagSet)) + LOG(' %s -> %s' % (firstOctet, v.__class__.__name__)) + LOG('new candidate ASN.1 spec is %s, chosen by %s' % (chosenSpec is None and '' or chosenSpec.prettyPrintType(), tagSet)) + elif tagSet == asn1Spec.tagSet or tagSet in asn1Spec.tagMap: chosenSpec = asn1Spec - if logger: - logger('candidate ASN.1 spec is %s' % asn1Spec.__class__.__name__) + if LOG: + LOG('candidate ASN.1 spec is %s' % asn1Spec.__class__.__name__) + else: chosenSpec = None @@ -1249,29 +1531,38 @@ class Decoder(object): try: # ambiguous type or just faster codec lookup concreteDecoder = typeMap[chosenSpec.typeId] - if logger: - logger('value decoder chosen for an ambiguous type by type ID %s' % (chosenSpec.typeId,)) + + if LOG: + LOG('value decoder chosen for an ambiguous type by type ID %s' % (chosenSpec.typeId,)) + except KeyError: # use base type for codec lookup to recover untagged types baseTagSet = tag.TagSet(chosenSpec.tagSet.baseTag, chosenSpec.tagSet.baseTag) try: # base type or tagged subtype concreteDecoder = tagMap[baseTagSet] - if logger: - logger('value decoder chosen by base %s' % (baseTagSet,)) + + if LOG: + LOG('value decoder chosen by base %s' % (baseTagSet,)) + except KeyError: concreteDecoder = None + if concreteDecoder: asn1Spec = chosenSpec state = stDecodeValue + else: state = stTryAsExplicitTag + else: concreteDecoder = None state = stTryAsExplicitTag - if logger: - logger('codec %s chosen by ASN.1 spec, decoding %s' % (state is stDecodeValue and concreteDecoder.__class__.__name__ or "", state is stDecodeValue and 'value' or 'as explicit tag')) + + if LOG: + LOG('codec %s chosen by ASN.1 spec, decoding %s' % (state is stDecodeValue and concreteDecoder.__class__.__name__ or "", state is stDecodeValue and 'value' or 'as explicit tag')) debug.scope.push(chosenSpec is None and '?' or chosenSpec.__class__.__name__) + if state is stDecodeValue: if not options.get('recursiveFlag', True) and not substrateFun: # deprecate this substrateFun = lambda a, b, c: (a, b[:c]) @@ -1285,6 +1576,7 @@ class Decoder(object): self, substrateFun, **options ) + else: value, substrate = concreteDecoder.valueDecoder( substrate, asn1Spec, @@ -1293,33 +1585,44 @@ class Decoder(object): **options ) - if logger: - logger('codec %s yields type %s, value:\n%s\n...remaining substrate is: %s' % (concreteDecoder.__class__.__name__, value.__class__.__name__, isinstance(value, base.Asn1Item) and value.prettyPrint() or value, substrate and debug.hexdump(substrate) or '')) + if LOG: + LOG('codec %s yields type %s, value:\n%s\n...remaining substrate is: %s' % (concreteDecoder.__class__.__name__, value.__class__.__name__, isinstance(value, base.Asn1Item) and value.prettyPrint() or value, substrate and debug.hexdump(substrate) or '')) state = stStop break + if state is stTryAsExplicitTag: - if tagSet and tagSet[0].tagFormat == tag.tagFormatConstructed and tagSet[0].tagClass != tag.tagClassUniversal: + if (tagSet and + tagSet[0].tagFormat == tag.tagFormatConstructed and + tagSet[0].tagClass != tag.tagClassUniversal): # Assume explicit tagging concreteDecoder = explicitTagDecoder state = stDecodeValue + else: concreteDecoder = None state = self.defaultErrorState - if logger: - logger('codec %s chosen, decoding %s' % (concreteDecoder and concreteDecoder.__class__.__name__ or "", state is stDecodeValue and 'value' or 'as failure')) + + if LOG: + LOG('codec %s chosen, decoding %s' % (concreteDecoder and concreteDecoder.__class__.__name__ or "", state is stDecodeValue and 'value' or 'as failure')) + if state is stDumpRawValue: concreteDecoder = self.defaultRawDecoder - if logger: - logger('codec %s chosen, decoding value' % concreteDecoder.__class__.__name__) + + if LOG: + LOG('codec %s chosen, decoding value' % concreteDecoder.__class__.__name__) + state = stDecodeValue + if state is stErrorCondition: raise error.PyAsn1Error( '%s not in asn1Spec: %r' % (tagSet, asn1Spec) ) - if logger: + + if LOG: debug.scope.pop() - logger('decoder left scope %s, call completed' % debug.scope) + LOG('decoder left scope %s, call completed' % debug.scope) + return value, substrate @@ -1349,7 +1652,7 @@ class Decoder(object): #: #: Raises #: ------ -#: :py:class:`~pyasn1.error.PyAsn1Error` +#: ~pyasn1.error.PyAsn1Error, ~pyasn1.error.SubstrateUnderrunError #: On decoding errors #: #: Examples diff --git a/server/www/packages/packages-linux/x64/pyasn1/codec/ber/encoder.py b/server/www/packages/packages-linux/x64/pyasn1/codec/ber/encoder.py index 0094b22..778aa86 100644 --- a/server/www/packages/packages-linux/x64/pyasn1/codec/ber/encoder.py +++ b/server/www/packages/packages-linux/x64/pyasn1/codec/ber/encoder.py @@ -1,9 +1,11 @@ # # This file is part of pyasn1 software. # -# Copyright (c) 2005-2018, Ilya Etingof +# Copyright (c) 2005-2019, Ilya Etingof # License: http://snmplabs.com/pyasn1/license.html # +import sys + from pyasn1 import debug from pyasn1 import error from pyasn1.codec.ber import eoo @@ -17,6 +19,8 @@ from pyasn1.type import useful __all__ = ['encode'] +LOG = debug.registerLoggee(__name__, flags=debug.DEBUG_ENCODER) + class AbstractItemEncoder(object): supportIndefLenMode = True @@ -31,29 +35,39 @@ class AbstractItemEncoder(object): encodedTag = tagClass | tagFormat if isConstructed: encodedTag |= tag.tagFormatConstructed + if tagId < 31: return encodedTag | tagId, + else: substrate = tagId & 0x7f, + tagId >>= 7 + while tagId: substrate = (0x80 | (tagId & 0x7f),) + substrate tagId >>= 7 + return (encodedTag | 0x1F,) + substrate def encodeLength(self, length, defMode): if not defMode and self.supportIndefLenMode: return (0x80,) + if length < 0x80: return length, + else: substrate = () while length: substrate = (length & 0xff,) + substrate length >>= 8 + substrateLen = len(substrate) + if substrateLen > 126: raise error.PyAsn1Error('Length octets overflow (%d)' % substrateLen) + return (0x80 | substrateLen,) + substrate def encodeValue(self, value, asn1Spec, encodeFun, **options): @@ -75,26 +89,51 @@ class AbstractItemEncoder(object): defMode = options.get('defMode', True) + substrate = null + for idx, singleTag in enumerate(tagSet.superTags): defModeOverride = defMode # base tag? if not idx: - substrate, isConstructed, isOctets = self.encodeValue( - value, asn1Spec, encodeFun, **options - ) + try: + substrate, isConstructed, isOctets = self.encodeValue( + value, asn1Spec, encodeFun, **options + ) + + except error.PyAsn1Error: + exc = sys.exc_info() + raise error.PyAsn1Error( + 'Error encoding %r: %s' % (value, exc[1])) + + if LOG: + LOG('encoded %svalue %s into %s' % ( + isConstructed and 'constructed ' or '', value, substrate + )) if not substrate and isConstructed and options.get('ifNotEmpty', False): return substrate - # primitive form implies definite mode if not isConstructed: defModeOverride = True + if LOG: + LOG('overridden encoding mode into definitive for primitive type') + header = self.encodeTag(singleTag, isConstructed) + + if LOG: + LOG('encoded %stag %s into %s' % ( + isConstructed and 'constructed ' or '', + singleTag, debug.hexdump(ints2octs(header)))) + header += self.encodeLength(len(substrate), defModeOverride) + if LOG: + LOG('encoded %s octets (tag + payload) into %s' % ( + len(substrate), debug.hexdump(ints2octs(header)))) + if isOctets: substrate = ints2octs(header) + substrate @@ -131,6 +170,11 @@ class IntegerEncoder(AbstractItemEncoder): def encodeValue(self, value, asn1Spec, encodeFun, **options): if value == 0: + if LOG: + LOG('encoding %spayload for zero INTEGER' % ( + self.supportCompactZero and 'no ' or '' + )) + # de-facto way to encode zero if self.supportCompactZero: return (), False, False @@ -157,11 +201,15 @@ class BitStringEncoder(AbstractItemEncoder): substrate = alignedValue.asOctets() return int2oct(len(substrate) * 8 - valueLength) + substrate, False, True + if LOG: + LOG('encoding into up to %s-octet chunks' % maxChunkSize) + baseTag = value.tagSet.baseTag # strip off explicit tags if baseTag: tagSet = tag.TagSet(baseTag, baseTag) + else: tagSet = tag.TagSet() @@ -195,44 +243,47 @@ class OctetStringEncoder(AbstractItemEncoder): if not maxChunkSize or len(substrate) <= maxChunkSize: return substrate, False, True - else: + if LOG: + LOG('encoding into up to %s-octet chunks' % maxChunkSize) - # strip off explicit tags for inner chunks + # strip off explicit tags for inner chunks - if asn1Spec is None: - baseTag = value.tagSet.baseTag + if asn1Spec is None: + baseTag = value.tagSet.baseTag - # strip off explicit tags - if baseTag: - tagSet = tag.TagSet(baseTag, baseTag) - else: - tagSet = tag.TagSet() + # strip off explicit tags + if baseTag: + tagSet = tag.TagSet(baseTag, baseTag) - asn1Spec = value.clone(tagSet=tagSet) + else: + tagSet = tag.TagSet() - elif not isOctetsType(value): - baseTag = asn1Spec.tagSet.baseTag + asn1Spec = value.clone(tagSet=tagSet) - # strip off explicit tags - if baseTag: - tagSet = tag.TagSet(baseTag, baseTag) - else: - tagSet = tag.TagSet() + elif not isOctetsType(value): + baseTag = asn1Spec.tagSet.baseTag - asn1Spec = asn1Spec.clone(tagSet=tagSet) + # strip off explicit tags + if baseTag: + tagSet = tag.TagSet(baseTag, baseTag) - pos = 0 - substrate = null + else: + tagSet = tag.TagSet() - while True: - chunk = value[pos:pos + maxChunkSize] - if not chunk: - break + asn1Spec = asn1Spec.clone(tagSet=tagSet) - substrate += encodeFun(chunk, asn1Spec, **options) - pos += maxChunkSize + pos = 0 + substrate = null - return substrate, True, True + while True: + chunk = value[pos:pos + maxChunkSize] + if not chunk: + break + + substrate += encodeFun(chunk, asn1Spec, **options) + pos += maxChunkSize + + return substrate, True, True class NullEncoder(AbstractItemEncoder): @@ -268,8 +319,10 @@ class ObjectIdentifierEncoder(AbstractItemEncoder): oid = (second + 80,) + oid[2:] else: raise error.PyAsn1Error('Impossible first/second arcs at %s' % (value,)) + elif first == 2: oid = (second + 80,) + oid[2:] + else: raise error.PyAsn1Error('Impossible first/second arcs at %s' % (value,)) @@ -280,15 +333,19 @@ class ObjectIdentifierEncoder(AbstractItemEncoder): if 0 <= subOid <= 127: # Optimize for the common case octets += (subOid,) + elif subOid > 127: # Pack large Sub-Object IDs res = (subOid & 0x7f,) subOid >>= 7 + while subOid: res = (0x80 | (subOid & 0x7f),) + res subOid >>= 7 + # Add packed Sub-Object ID to resulted Object ID octets += res + else: raise error.PyAsn1Error('Negative OID arc %s at %s' % (subOid, value)) @@ -304,12 +361,16 @@ class RealEncoder(AbstractItemEncoder): ms, es = 1, 1 if m < 0: ms = -1 # mantissa sign + if e < 0: - es = -1 # exponenta sign + es = -1 # exponent sign + m *= ms + if encbase == 8: m *= 2 ** (abs(e) % 3 * es) e = abs(e) // 3 * es + elif encbase == 16: m *= 2 ** (abs(e) % 4 * es) e = abs(e) // 4 * es @@ -320,6 +381,7 @@ class RealEncoder(AbstractItemEncoder): e -= 1 continue break + return ms, int(m), encbase, e def _chooseEncBase(self, value): @@ -327,23 +389,32 @@ class RealEncoder(AbstractItemEncoder): encBase = [2, 8, 16] if value.binEncBase in encBase: return self._dropFloatingPoint(m, value.binEncBase, e) + elif self.binEncBase in encBase: return self._dropFloatingPoint(m, self.binEncBase, e) - # auto choosing base 2/8/16 + + # auto choosing base 2/8/16 mantissa = [m, m, m] - exponenta = [e, e, e] + exponent = [e, e, e] sign = 1 encbase = 2 e = float('inf') + for i in range(3): (sign, mantissa[i], encBase[i], - exponenta[i]) = self._dropFloatingPoint(mantissa[i], encBase[i], exponenta[i]) - if abs(exponenta[i]) < abs(e) or (abs(exponenta[i]) == abs(e) and mantissa[i] < m): - e = exponenta[i] + exponent[i]) = self._dropFloatingPoint(mantissa[i], encBase[i], exponent[i]) + + if abs(exponent[i]) < abs(e) or (abs(exponent[i]) == abs(e) and mantissa[i] < m): + e = exponent[i] m = int(mantissa[i]) encbase = encBase[i] + + if LOG: + LOG('automatically chosen REAL encoding base %s, sign %s, mantissa %s, ' + 'exponent %s' % (encbase, sign, m, e)) + return sign, m, encbase, e def encodeValue(self, value, asn1Spec, encodeFun, **options): @@ -352,69 +423,98 @@ class RealEncoder(AbstractItemEncoder): if value.isPlusInf: return (0x40,), False, False + if value.isMinusInf: return (0x41,), False, False + m, b, e = value + if not m: return null, False, True + if b == 10: + if LOG: + LOG('encoding REAL into character form') + return str2octs('\x03%dE%s%d' % (m, e == 0 and '+' or '', e)), False, True + elif b == 2: fo = 0x80 # binary encoding ms, m, encbase, e = self._chooseEncBase(value) + if ms < 0: # mantissa sign fo |= 0x40 # sign bit - # exponenta & mantissa normalization + + # exponent & mantissa normalization if encbase == 2: while m & 0x1 == 0: m >>= 1 e += 1 + elif encbase == 8: while m & 0x7 == 0: m >>= 3 e += 1 fo |= 0x10 + else: # encbase = 16 while m & 0xf == 0: m >>= 4 e += 1 fo |= 0x20 + sf = 0 # scale factor + while m & 0x1 == 0: m >>= 1 sf += 1 + if sf > 3: raise error.PyAsn1Error('Scale factor overflow') # bug if raised + fo |= sf << 2 eo = null if e == 0 or e == -1: eo = int2oct(e & 0xff) + else: while e not in (0, -1): eo = int2oct(e & 0xff) + eo e >>= 8 + if e == 0 and eo and oct2int(eo[0]) & 0x80: eo = int2oct(0) + eo + if e == -1 and eo and not (oct2int(eo[0]) & 0x80): eo = int2oct(0xff) + eo + n = len(eo) if n > 0xff: raise error.PyAsn1Error('Real exponent overflow') + if n == 1: pass + elif n == 2: fo |= 1 + elif n == 3: fo |= 2 + else: fo |= 3 eo = int2oct(n & 0xff) + eo + po = null + while m: po = int2oct(m & 0xff) + po m >>= 8 + substrate = int2oct(fo) + eo + po + return substrate, False, True + else: raise error.PyAsn1Error('Prohibited Real base %s' % b) @@ -428,9 +528,18 @@ class SequenceEncoder(AbstractItemEncoder): substrate = null + omitEmptyOptionals = options.get( + 'omitEmptyOptionals', self.omitEmptyOptionals) + + if LOG: + LOG('%sencoding empty OPTIONAL components' % ( + omitEmptyOptionals and 'not ' or '')) + if asn1Spec is None: # instance of ASN.1 schema - value.verifySizeSpec() + inconsistency = value.isInconsistent + if inconsistency: + raise inconsistency namedTypes = value.componentType @@ -439,23 +548,44 @@ class SequenceEncoder(AbstractItemEncoder): namedType = namedTypes[idx] if namedType.isOptional and not component.isValue: - continue + if LOG: + LOG('not encoding OPTIONAL component %r' % (namedType,)) + continue if namedType.isDefaulted and component == namedType.asn1Object: - continue + if LOG: + LOG('not encoding DEFAULT component %r' % (namedType,)) + continue - if self.omitEmptyOptionals: + if omitEmptyOptionals: options.update(ifNotEmpty=namedType.isOptional) - chunk = encodeFun(component, asn1Spec, **options) - # wrap open type blob if needed if namedTypes and namedType.openType: - wrapType = namedType.asn1Object - if wrapType.tagSet and not wrapType.isSameTypeWith(component): - chunk = encodeFun(chunk, wrapType, **options) - substrate += chunk + wrapType = namedType.asn1Object + + if wrapType.typeId in ( + univ.SetOf.typeId, univ.SequenceOf.typeId): + + substrate += encodeFun( + component, asn1Spec, + **dict(options, wrapType=wrapType.componentType)) + + else: + chunk = encodeFun(component, asn1Spec, **options) + + if wrapType.isSameTypeWith(component): + substrate += chunk + + else: + substrate += encodeFun(chunk, wrapType, **options) + + if LOG: + LOG('wrapped with wrap type %r' % (wrapType,)) + + else: + substrate += encodeFun(component, asn1Spec, **options) else: # bare Python value + ASN.1 schema @@ -465,43 +595,87 @@ class SequenceEncoder(AbstractItemEncoder): component = value[namedType.name] except KeyError: - raise error.PyAsn1Error('Component name "%s" not found in %r' % (namedType.name, value)) + raise error.PyAsn1Error('Component name "%s" not found in %r' % ( + namedType.name, value)) if namedType.isOptional and namedType.name not in value: + if LOG: + LOG('not encoding OPTIONAL component %r' % (namedType,)) continue if namedType.isDefaulted and component == namedType.asn1Object: + if LOG: + LOG('not encoding DEFAULT component %r' % (namedType,)) continue - if self.omitEmptyOptionals: + if omitEmptyOptionals: options.update(ifNotEmpty=namedType.isOptional) - chunk = encodeFun(component, asn1Spec[idx], **options) + componentSpec = namedType.asn1Object # wrap open type blob if needed if namedType.openType: - wrapType = namedType.asn1Object - if wrapType.tagSet and not wrapType.isSameTypeWith(component): - chunk = encodeFun(chunk, wrapType, **options) - substrate += chunk + if componentSpec.typeId in ( + univ.SetOf.typeId, univ.SequenceOf.typeId): + + substrate += encodeFun( + component, componentSpec, + **dict(options, wrapType=componentSpec.componentType)) + + else: + chunk = encodeFun(component, componentSpec, **options) + + if componentSpec.isSameTypeWith(component): + substrate += chunk + + else: + substrate += encodeFun(chunk, componentSpec, **options) + + if LOG: + LOG('wrapped with wrap type %r' % (componentSpec,)) + + else: + substrate += encodeFun(component, componentSpec, **options) return substrate, True, True class SequenceOfEncoder(AbstractItemEncoder): - def encodeValue(self, value, asn1Spec, encodeFun, **options): + def _encodeComponents(self, value, asn1Spec, encodeFun, **options): + if asn1Spec is None: - value.verifySizeSpec() + inconsistency = value.isInconsistent + if inconsistency: + raise inconsistency + else: asn1Spec = asn1Spec.componentType - substrate = null + chunks = [] + + wrapType = options.pop('wrapType', None) for idx, component in enumerate(value): - substrate += encodeFun(value[idx], asn1Spec, **options) + chunk = encodeFun(component, asn1Spec, **options) - return substrate, True, True + if (wrapType is not None and + not wrapType.isSameTypeWith(component)): + # wrap encoded value with wrapper container (e.g. ANY) + chunk = encodeFun(chunk, wrapType, **options) + + if LOG: + LOG('wrapped with wrap type %r' % (wrapType,)) + + chunks.append(chunk) + + return chunks + + def encodeValue(self, value, asn1Spec, encodeFun, **options): + chunks = self._encodeComponents( + value, asn1Spec, encodeFun, **options) + + return null.join(chunks), True, True class ChoiceEncoder(AbstractItemEncoder): @@ -620,13 +794,8 @@ class Encoder(object): raise error.PyAsn1Error('Value %r is not ASN.1 type instance ' 'and "asn1Spec" not given' % (value,)) - if debug.logger & debug.flagEncoder: - logger = debug.logger - else: - logger = None - - if logger: - logger('encoder called in %sdef mode, chunk size %s for ' + if LOG: + LOG('encoder called in %sdef mode, chunk size %s for ' 'type %s, value:\n%s' % (not options.get('defMode', True) and 'in' or '', options.get('maxChunkSize', 0), asn1Spec is None and value.prettyPrintType() or asn1Spec.prettyPrintType(), value)) if self.fixedDefLengthMode is not None: @@ -639,8 +808,8 @@ class Encoder(object): try: concreteEncoder = self.__typeMap[typeId] - if logger: - logger('using value codec %s chosen by type ID %s' % (concreteEncoder.__class__.__name__, typeId)) + if LOG: + LOG('using value codec %s chosen by type ID %s' % (concreteEncoder.__class__.__name__, typeId)) except KeyError: if asn1Spec is None: @@ -657,13 +826,13 @@ class Encoder(object): except KeyError: raise error.PyAsn1Error('No encoder for %r (%s)' % (value, tagSet)) - if logger: - logger('using value codec %s chosen by tagSet %s' % (concreteEncoder.__class__.__name__, tagSet)) + if LOG: + LOG('using value codec %s chosen by tagSet %s' % (concreteEncoder.__class__.__name__, tagSet)) substrate = concreteEncoder.encode(value, asn1Spec, self, **options) - if logger: - logger('codec %s built %s octets of substrate: %s\nencoder completed' % (concreteEncoder, len(substrate), debug.hexdump(substrate))) + if LOG: + LOG('codec %s built %s octets of substrate: %s\nencoder completed' % (concreteEncoder, len(substrate), debug.hexdump(substrate))) return substrate @@ -684,7 +853,7 @@ class Encoder(object): #: Optional ASN.1 schema or value object e.g. :py:class:`~pyasn1.type.base.PyAsn1Item` derivative #: #: defMode: :py:class:`bool` -#: If `False`, produces indefinite length encoding +#: If :obj:`False`, produces indefinite length encoding #: #: maxChunkSize: :py:class:`int` #: Maximum chunk size in chunked encoding mode (0 denotes unlimited chunk size) @@ -696,7 +865,7 @@ class Encoder(object): #: #: Raises #: ------ -#: :py:class:`~pyasn1.error.PyAsn1Error` +#: ~pyasn1.error.PyAsn1Error #: On encoding errors #: #: Examples diff --git a/server/www/packages/packages-linux/x64/pyasn1/codec/ber/eoo.py b/server/www/packages/packages-linux/x64/pyasn1/codec/ber/eoo.py index d4cd827..48eb859 100644 --- a/server/www/packages/packages-linux/x64/pyasn1/codec/ber/eoo.py +++ b/server/www/packages/packages-linux/x64/pyasn1/codec/ber/eoo.py @@ -1,7 +1,7 @@ # # This file is part of pyasn1 software. # -# Copyright (c) 2005-2018, Ilya Etingof +# Copyright (c) 2005-2019, Ilya Etingof # License: http://snmplabs.com/pyasn1/license.html # from pyasn1.type import base @@ -10,7 +10,7 @@ from pyasn1.type import tag __all__ = ['endOfOctets'] -class EndOfOctets(base.AbstractSimpleAsn1Item): +class EndOfOctets(base.SimpleAsn1Type): defaultValue = 0 tagSet = tag.initTagSet( tag.Tag(tag.tagClassUniversal, tag.tagFormatSimple, 0x00) diff --git a/server/www/packages/packages-linux/x64/pyasn1/codec/cer/decoder.py b/server/www/packages/packages-linux/x64/pyasn1/codec/cer/decoder.py index 66572ec..3e86fd0 100644 --- a/server/www/packages/packages-linux/x64/pyasn1/codec/cer/decoder.py +++ b/server/www/packages/packages-linux/x64/pyasn1/codec/cer/decoder.py @@ -1,7 +1,7 @@ # # This file is part of pyasn1 software. # -# Copyright (c) 2005-2018, Ilya Etingof +# Copyright (c) 2005-2019, Ilya Etingof # License: http://snmplabs.com/pyasn1/license.html # from pyasn1 import error @@ -87,7 +87,7 @@ class Decoder(decoder.Decoder): #: #: Raises #: ------ -#: :py:class:`~pyasn1.error.PyAsn1Error` +#: ~pyasn1.error.PyAsn1Error, ~pyasn1.error.SubstrateUnderrunError #: On decoding errors #: #: Examples diff --git a/server/www/packages/packages-linux/x64/pyasn1/codec/cer/encoder.py b/server/www/packages/packages-linux/x64/pyasn1/codec/cer/encoder.py index 768d3c1..935b696 100644 --- a/server/www/packages/packages-linux/x64/pyasn1/codec/cer/encoder.py +++ b/server/www/packages/packages-linux/x64/pyasn1/codec/cer/encoder.py @@ -1,7 +1,7 @@ # # This file is part of pyasn1 software. # -# Copyright (c) 2005-2018, Ilya Etingof +# Copyright (c) 2005-2019, Ilya Etingof # License: http://snmplabs.com/pyasn1/license.html # from pyasn1 import error @@ -31,17 +31,20 @@ class RealEncoder(encoder.RealEncoder): # specialized GeneralStringEncoder here class TimeEncoderMixIn(object): - zchar, = str2octs('Z') - pluschar, = str2octs('+') - minuschar, = str2octs('-') - commachar, = str2octs(',') - minLength = 12 - maxLength = 19 + Z_CHAR = ord('Z') + PLUS_CHAR = ord('+') + MINUS_CHAR = ord('-') + COMMA_CHAR = ord(',') + DOT_CHAR = ord('.') + ZERO_CHAR = ord('0') + + MIN_LENGTH = 12 + MAX_LENGTH = 19 def encodeValue(self, value, asn1Spec, encodeFun, **options): - # Encoding constraints: + # CER encoding constraints: # - minutes are mandatory, seconds are optional - # - subseconds must NOT be zero + # - sub-seconds must NOT be zero / no meaningless zeros # - no hanging fraction dot # - time in UTC (Z) # - only dot is allowed for fractions @@ -49,20 +52,46 @@ class TimeEncoderMixIn(object): if asn1Spec is not None: value = asn1Spec.clone(value) - octets = value.asOctets() + numbers = value.asNumbers() - if not self.minLength < len(octets) < self.maxLength: - raise error.PyAsn1Error('Length constraint violated: %r' % value) + if self.PLUS_CHAR in numbers or self.MINUS_CHAR in numbers: + raise error.PyAsn1Error('Must be UTC time: %r' % value) - if self.pluschar in octets or self.minuschar in octets: - raise error.PyAsn1Error('Must be UTC time: %r' % octets) + if numbers[-1] != self.Z_CHAR: + raise error.PyAsn1Error('Missing "Z" time zone specifier: %r' % value) - if octets[-1] != self.zchar: - raise error.PyAsn1Error('Missing "Z" time zone specifier: %r' % octets) - - if self.commachar in octets: + if self.COMMA_CHAR in numbers: raise error.PyAsn1Error('Comma in fractions disallowed: %r' % value) + if self.DOT_CHAR in numbers: + + isModified = False + + numbers = list(numbers) + + searchIndex = min(numbers.index(self.DOT_CHAR) + 4, len(numbers) - 1) + + while numbers[searchIndex] != self.DOT_CHAR: + if numbers[searchIndex] == self.ZERO_CHAR: + del numbers[searchIndex] + isModified = True + + searchIndex -= 1 + + searchIndex += 1 + + if searchIndex < len(numbers): + if numbers[searchIndex] == self.Z_CHAR: + # drop hanging comma + del numbers[searchIndex - 1] + isModified = True + + if isModified: + value = value.clone(numbers) + + if not self.MIN_LENGTH < len(numbers) < self.MAX_LENGTH: + raise error.PyAsn1Error('Length constraint violated: %r' % value) + options.update(maxChunkSize=1000) return encoder.OctetStringEncoder.encodeValue( @@ -71,13 +100,44 @@ class TimeEncoderMixIn(object): class GeneralizedTimeEncoder(TimeEncoderMixIn, encoder.OctetStringEncoder): - minLength = 12 - maxLength = 19 + MIN_LENGTH = 12 + MAX_LENGTH = 20 class UTCTimeEncoder(TimeEncoderMixIn, encoder.OctetStringEncoder): - minLength = 10 - maxLength = 14 + MIN_LENGTH = 10 + MAX_LENGTH = 14 + + +class SetOfEncoder(encoder.SequenceOfEncoder): + def encodeValue(self, value, asn1Spec, encodeFun, **options): + chunks = self._encodeComponents( + value, asn1Spec, encodeFun, **options) + + # sort by serialised and padded components + if len(chunks) > 1: + zero = str2octs('\x00') + maxLen = max(map(len, chunks)) + paddedChunks = [ + (x.ljust(maxLen, zero), x) for x in chunks + ] + paddedChunks.sort(key=lambda x: x[0]) + + chunks = [x[1] for x in paddedChunks] + + return null.join(chunks), True, True + + +class SequenceOfEncoder(encoder.SequenceOfEncoder): + def encodeValue(self, value, asn1Spec, encodeFun, **options): + + if options.get('ifNotEmpty', False) and not len(value): + return null, True, True + + chunks = self._encodeComponents( + value, asn1Spec, encodeFun, **options) + + return null.join(chunks), True, True class SetEncoder(encoder.SequenceEncoder): @@ -109,7 +169,9 @@ class SetEncoder(encoder.SequenceEncoder): if asn1Spec is None: # instance of ASN.1 schema - value.verifySizeSpec() + inconsistency = value.isInconsistent + if inconsistency: + raise inconsistency namedTypes = value.componentType @@ -168,55 +230,10 @@ class SetEncoder(encoder.SequenceEncoder): return substrate, True, True -class SetOfEncoder(encoder.SequenceOfEncoder): - def encodeValue(self, value, asn1Spec, encodeFun, **options): - if asn1Spec is None: - value.verifySizeSpec() - else: - asn1Spec = asn1Spec.componentType - - components = [encodeFun(x, asn1Spec, **options) - for x in value] - - # sort by serialised and padded components - if len(components) > 1: - zero = str2octs('\x00') - maxLen = max(map(len, components)) - paddedComponents = [ - (x.ljust(maxLen, zero), x) for x in components - ] - paddedComponents.sort(key=lambda x: x[0]) - - components = [x[1] for x in paddedComponents] - - substrate = null.join(components) - - return substrate, True, True - - class SequenceEncoder(encoder.SequenceEncoder): omitEmptyOptionals = True -class SequenceOfEncoder(encoder.SequenceOfEncoder): - def encodeValue(self, value, asn1Spec, encodeFun, **options): - - if options.get('ifNotEmpty', False) and not len(value): - return null, True, True - - if asn1Spec is None: - value.verifySizeSpec() - else: - asn1Spec = asn1Spec.componentType - - substrate = null - - for idx, component in enumerate(value): - substrate += encodeFun(value[idx], asn1Spec, **options) - - return substrate, True, True - - tagMap = encoder.tagMap.copy() tagMap.update({ univ.Boolean.tagSet: BooleanEncoder(), @@ -269,7 +286,7 @@ class Encoder(encoder.Encoder): #: #: Raises #: ------ -#: :py:class:`~pyasn1.error.PyAsn1Error` +#: ~pyasn1.error.PyAsn1Error #: On encoding errors #: #: Examples diff --git a/server/www/packages/packages-linux/x64/pyasn1/codec/der/decoder.py b/server/www/packages/packages-linux/x64/pyasn1/codec/der/decoder.py index f67d025..1a13fdb 100644 --- a/server/www/packages/packages-linux/x64/pyasn1/codec/der/decoder.py +++ b/server/www/packages/packages-linux/x64/pyasn1/codec/der/decoder.py @@ -1,7 +1,7 @@ # # This file is part of pyasn1 software. # -# Copyright (c) 2005-2018, Ilya Etingof +# Copyright (c) 2005-2019, Ilya Etingof # License: http://snmplabs.com/pyasn1/license.html # from pyasn1.codec.cer import decoder @@ -67,7 +67,7 @@ class Decoder(decoder.Decoder): #: #: Raises #: ------ -#: :py:class:`~pyasn1.error.PyAsn1Error` +#: ~pyasn1.error.PyAsn1Error, ~pyasn1.error.SubstrateUnderrunError #: On decoding errors #: #: Examples diff --git a/server/www/packages/packages-linux/x64/pyasn1/codec/der/encoder.py b/server/www/packages/packages-linux/x64/pyasn1/codec/der/encoder.py index 756d9fe..90e982d 100644 --- a/server/www/packages/packages-linux/x64/pyasn1/codec/der/encoder.py +++ b/server/www/packages/packages-linux/x64/pyasn1/codec/der/encoder.py @@ -1,7 +1,7 @@ # # This file is part of pyasn1 software. # -# Copyright (c) 2005-2018, Ilya Etingof +# Copyright (c) 2005-2019, Ilya Etingof # License: http://snmplabs.com/pyasn1/license.html # from pyasn1 import error @@ -82,7 +82,7 @@ class Encoder(encoder.Encoder): #: #: Raises #: ------ -#: :py:class:`~pyasn1.error.PyAsn1Error` +#: ~pyasn1.error.PyAsn1Error #: On encoding errors #: #: Examples diff --git a/server/www/packages/packages-linux/x64/pyasn1/codec/native/decoder.py b/server/www/packages/packages-linux/x64/pyasn1/codec/native/decoder.py index 78fcda6..104b92e 100644 --- a/server/www/packages/packages-linux/x64/pyasn1/codec/native/decoder.py +++ b/server/www/packages/packages-linux/x64/pyasn1/codec/native/decoder.py @@ -1,7 +1,7 @@ # # This file is part of pyasn1 software. # -# Copyright (c) 2005-2018, Ilya Etingof +# Copyright (c) 2005-2019, Ilya Etingof # License: http://snmplabs.com/pyasn1/license.html # from pyasn1 import debug @@ -14,6 +14,8 @@ from pyasn1.type import useful __all__ = ['decode'] +LOG = debug.registerLoggee(__name__, flags=debug.DEBUG_DECODER) + class AbstractScalarDecoder(object): def __call__(self, pyObject, asn1Spec, decodeFun=None, **options): @@ -136,13 +138,10 @@ class Decoder(object): self.__typeMap = typeMap def __call__(self, pyObject, asn1Spec, **options): - if debug.logger & debug.flagDecoder: - logger = debug.logger - else: - logger = None - if logger: + + if LOG: debug.scope.push(type(pyObject).__name__) - logger('decoder called at scope %s, working with type %s' % (debug.scope, type(pyObject).__name__)) + LOG('decoder called at scope %s, working with type %s' % (debug.scope, type(pyObject).__name__)) if asn1Spec is None or not isinstance(asn1Spec, base.Asn1Item): raise error.PyAsn1Error('asn1Spec is not valid (should be an instance of an ASN.1 Item, not %s)' % asn1Spec.__class__.__name__) @@ -159,13 +158,13 @@ class Decoder(object): except KeyError: raise error.PyAsn1Error('Unknown ASN.1 tag %s' % asn1Spec.tagSet) - if logger: - logger('calling decoder %s on Python type %s <%s>' % (type(valueDecoder).__name__, type(pyObject).__name__, repr(pyObject))) + if LOG: + LOG('calling decoder %s on Python type %s <%s>' % (type(valueDecoder).__name__, type(pyObject).__name__, repr(pyObject))) value = valueDecoder(pyObject, asn1Spec, self, **options) - if logger: - logger('decoder %s produced ASN.1 type %s <%s>' % (type(valueDecoder).__name__, type(value).__name__, repr(value))) + if LOG: + LOG('decoder %s produced ASN.1 type %s <%s>' % (type(valueDecoder).__name__, type(value).__name__, repr(value))) debug.scope.pop() return value @@ -196,7 +195,7 @@ class Decoder(object): #: #: Raises #: ------ -#: :py:class:`~pyasn1.error.PyAsn1Error` +#: ~pyasn1.error.PyAsn1Error #: On decoding errors #: #: Examples diff --git a/server/www/packages/packages-linux/x64/pyasn1/codec/native/encoder.py b/server/www/packages/packages-linux/x64/pyasn1/codec/native/encoder.py index 0956191..4318abd 100644 --- a/server/www/packages/packages-linux/x64/pyasn1/codec/native/encoder.py +++ b/server/www/packages/packages-linux/x64/pyasn1/codec/native/encoder.py @@ -1,7 +1,7 @@ # # This file is part of pyasn1 software. # -# Copyright (c) 2005-2018, Ilya Etingof +# Copyright (c) 2005-2019, Ilya Etingof # License: http://snmplabs.com/pyasn1/license.html # try: @@ -20,6 +20,8 @@ from pyasn1.type import useful __all__ = ['encode'] +LOG = debug.registerLoggee(__name__, flags=debug.DEBUG_ENCODER) + class AbstractItemEncoder(object): def encode(self, value, encodeFun, **options): @@ -70,7 +72,9 @@ class SetEncoder(AbstractItemEncoder): protoDict = dict def encode(self, value, encodeFun, **options): - value.verifySizeSpec() + inconsistency = value.isInconsistent + if inconsistency: + raise inconsistency namedTypes = value.componentType substrate = self.protoDict() @@ -88,7 +92,9 @@ class SequenceEncoder(SetEncoder): class SequenceOfEncoder(AbstractItemEncoder): def encode(self, value, encodeFun, **options): - value.verifySizeSpec() + inconsistency = value.isInconsistent + if inconsistency: + raise inconsistency return [encodeFun(x, **options) for x in value] @@ -180,14 +186,9 @@ class Encoder(object): if not isinstance(value, base.Asn1Item): raise error.PyAsn1Error('value is not valid (should be an instance of an ASN.1 Item)') - if debug.logger & debug.flagEncoder: - logger = debug.logger - else: - logger = None - - if logger: + if LOG: debug.scope.push(type(value).__name__) - logger('encoder called for type %s <%s>' % (type(value).__name__, value.prettyPrint())) + LOG('encoder called for type %s <%s>' % (type(value).__name__, value.prettyPrint())) tagSet = value.tagSet @@ -204,13 +205,13 @@ class Encoder(object): except KeyError: raise error.PyAsn1Error('No encoder for %s' % (value,)) - if logger: - logger('using value codec %s chosen by %s' % (concreteEncoder.__class__.__name__, tagSet)) + if LOG: + LOG('using value codec %s chosen by %s' % (concreteEncoder.__class__.__name__, tagSet)) pyObject = concreteEncoder.encode(value, self, **options) - if logger: - logger('encoder %s produced: %s' % (type(concreteEncoder).__name__, repr(pyObject))) + if LOG: + LOG('encoder %s produced: %s' % (type(concreteEncoder).__name__, repr(pyObject))) debug.scope.pop() return pyObject @@ -238,7 +239,7 @@ class Encoder(object): #: #: Raises #: ------ -#: :py:class:`~pyasn1.error.PyAsn1Error` +#: ~pyasn1.error.PyAsn1Error #: On encoding errors #: #: Examples diff --git a/server/www/packages/packages-linux/x64/pyasn1/compat/binary.py b/server/www/packages/packages-linux/x64/pyasn1/compat/binary.py index c38a650..addbdc9 100644 --- a/server/www/packages/packages-linux/x64/pyasn1/compat/binary.py +++ b/server/www/packages/packages-linux/x64/pyasn1/compat/binary.py @@ -1,7 +1,7 @@ # # This file is part of pyasn1 software. # -# Copyright (c) 2005-2018, Ilya Etingof +# Copyright (c) 2005-2019, Ilya Etingof # License: http://snmplabs.com/pyasn1/license.html # from sys import version_info diff --git a/server/www/packages/packages-linux/x64/pyasn1/compat/calling.py b/server/www/packages/packages-linux/x64/pyasn1/compat/calling.py index c60b50d..778a3d1 100644 --- a/server/www/packages/packages-linux/x64/pyasn1/compat/calling.py +++ b/server/www/packages/packages-linux/x64/pyasn1/compat/calling.py @@ -1,7 +1,7 @@ # # This file is part of pyasn1 software. # -# Copyright (c) 2005-2018, Ilya Etingof +# Copyright (c) 2005-2019, Ilya Etingof # License: http://snmplabs.com/pyasn1/license.html # from sys import version_info diff --git a/server/www/packages/packages-linux/x64/pyasn1/compat/dateandtime.py b/server/www/packages/packages-linux/x64/pyasn1/compat/dateandtime.py index 27526ad..5e471bf 100644 --- a/server/www/packages/packages-linux/x64/pyasn1/compat/dateandtime.py +++ b/server/www/packages/packages-linux/x64/pyasn1/compat/dateandtime.py @@ -1,7 +1,7 @@ # # This file is part of pyasn1 software. # -# Copyright (c) 2005-2018, Ilya Etingof +# Copyright (c) 2005-2019, Ilya Etingof # License: http://snmplabs.com/pyasn1/license.html # import time diff --git a/server/www/packages/packages-linux/x64/pyasn1/compat/integer.py b/server/www/packages/packages-linux/x64/pyasn1/compat/integer.py index bb3d099..4b31791 100644 --- a/server/www/packages/packages-linux/x64/pyasn1/compat/integer.py +++ b/server/www/packages/packages-linux/x64/pyasn1/compat/integer.py @@ -1,7 +1,7 @@ # # This file is part of pyasn1 software. # -# Copyright (c) 2005-2018, Ilya Etingof +# Copyright (c) 2005-2019, Ilya Etingof # License: http://snmplabs.com/pyasn1/license.html # import sys diff --git a/server/www/packages/packages-linux/x64/pyasn1/compat/octets.py b/server/www/packages/packages-linux/x64/pyasn1/compat/octets.py index a06db5d..99d23bb 100644 --- a/server/www/packages/packages-linux/x64/pyasn1/compat/octets.py +++ b/server/www/packages/packages-linux/x64/pyasn1/compat/octets.py @@ -1,7 +1,7 @@ # # This file is part of pyasn1 software. # -# Copyright (c) 2005-2018, Ilya Etingof +# Copyright (c) 2005-2019, Ilya Etingof # License: http://snmplabs.com/pyasn1/license.html # from sys import version_info diff --git a/server/www/packages/packages-linux/x64/pyasn1/compat/string.py b/server/www/packages/packages-linux/x64/pyasn1/compat/string.py index 4d8a045..b9bc8c3 100644 --- a/server/www/packages/packages-linux/x64/pyasn1/compat/string.py +++ b/server/www/packages/packages-linux/x64/pyasn1/compat/string.py @@ -1,7 +1,7 @@ # # This file is part of pyasn1 software. # -# Copyright (c) 2005-2018, Ilya Etingof +# Copyright (c) 2005-2019, Ilya Etingof # License: http://snmplabs.com/pyasn1/license.html # from sys import version_info diff --git a/server/www/packages/packages-linux/x64/pyasn1/debug.py b/server/www/packages/packages-linux/x64/pyasn1/debug.py index ab72fa8..8707aa8 100644 --- a/server/www/packages/packages-linux/x64/pyasn1/debug.py +++ b/server/www/packages/packages-linux/x64/pyasn1/debug.py @@ -1,10 +1,11 @@ # # This file is part of pyasn1 software. # -# Copyright (c) 2005-2018, Ilya Etingof +# Copyright (c) 2005-2019, Ilya Etingof # License: http://snmplabs.com/pyasn1/license.html # import logging +import sys from pyasn1 import __version__ from pyasn1 import error @@ -12,18 +13,20 @@ from pyasn1.compat.octets import octs2ints __all__ = ['Debug', 'setLogger', 'hexdump'] -flagNone = 0x0000 -flagEncoder = 0x0001 -flagDecoder = 0x0002 -flagAll = 0xffff +DEBUG_NONE = 0x0000 +DEBUG_ENCODER = 0x0001 +DEBUG_DECODER = 0x0002 +DEBUG_ALL = 0xffff -flagMap = { - 'none': flagNone, - 'encoder': flagEncoder, - 'decoder': flagDecoder, - 'all': flagAll +FLAG_MAP = { + 'none': DEBUG_NONE, + 'encoder': DEBUG_ENCODER, + 'decoder': DEBUG_DECODER, + 'all': DEBUG_ALL } +LOGGEE_MAP = {} + class Printer(object): # noinspection PyShadowingNames @@ -66,7 +69,7 @@ class Debug(object): defaultPrinter = Printer() def __init__(self, *flags, **options): - self._flags = flagNone + self._flags = DEBUG_NONE if 'loggerName' in options: # route our logs to parent logger @@ -89,9 +92,9 @@ class Debug(object): flag = flag[1:] try: if inverse: - self._flags &= ~flagMap[flag] + self._flags &= ~FLAG_MAP[flag] else: - self._flags |= flagMap[flag] + self._flags |= FLAG_MAP[flag] except KeyError: raise error.PyAsn1Error('bad debug flag %s' % flag) @@ -109,17 +112,26 @@ class Debug(object): def __rand__(self, flag): return flag & self._flags - -logger = 0 +_LOG = DEBUG_NONE def setLogger(userLogger): - global logger + global _LOG if userLogger: - logger = userLogger + _LOG = userLogger else: - logger = 0 + _LOG = DEBUG_NONE + + # Update registered logging clients + for module, (name, flags) in LOGGEE_MAP.items(): + setattr(module, name, _LOG & flags and _LOG or DEBUG_NONE) + + +def registerLoggee(module, name='LOG', flags=DEBUG_NONE): + LOGGEE_MAP[sys.modules[module]] = name, flags + setLogger(_LOG) + return _LOG def hexdump(octets): diff --git a/server/www/packages/packages-linux/x64/pyasn1/error.py b/server/www/packages/packages-linux/x64/pyasn1/error.py index c05e65c..4f48db2 100644 --- a/server/www/packages/packages-linux/x64/pyasn1/error.py +++ b/server/www/packages/packages-linux/x64/pyasn1/error.py @@ -1,29 +1,75 @@ # # This file is part of pyasn1 software. # -# Copyright (c) 2005-2018, Ilya Etingof +# Copyright (c) 2005-2019, Ilya Etingof # License: http://snmplabs.com/pyasn1/license.html # class PyAsn1Error(Exception): - """Create pyasn1 exception object + """Base pyasn1 exception - The `PyAsn1Error` exception represents generic, usually fatal, error. + `PyAsn1Error` is the base exception class (based on + :class:`Exception`) that represents all possible ASN.1 related + errors. """ class ValueConstraintError(PyAsn1Error): - """Create pyasn1 exception object + """ASN.1 type constraints violation exception The `ValueConstraintError` exception indicates an ASN.1 value constraint violation. + + It might happen on value object instantiation (for scalar types) or on + serialization (for constructed types). """ class SubstrateUnderrunError(PyAsn1Error): - """Create pyasn1 exception object + """ASN.1 data structure deserialization error The `SubstrateUnderrunError` exception indicates insufficient serialised - data on input of a deserialisation routine. + data on input of a de-serialization codec. """ + + +class PyAsn1UnicodeError(PyAsn1Error, UnicodeError): + """Unicode text processing error + + The `PyAsn1UnicodeError` exception is a base class for errors relating to + unicode text de/serialization. + + Apart from inheriting from :class:`PyAsn1Error`, it also inherits from + :class:`UnicodeError` to help the caller catching unicode-related errors. + """ + def __init__(self, message, unicode_error=None): + if isinstance(unicode_error, UnicodeError): + UnicodeError.__init__(self, *unicode_error.args) + PyAsn1Error.__init__(self, message) + + +class PyAsn1UnicodeDecodeError(PyAsn1UnicodeError, UnicodeDecodeError): + """Unicode text decoding error + + The `PyAsn1UnicodeDecodeError` exception represents a failure to + deserialize unicode text. + + Apart from inheriting from :class:`PyAsn1UnicodeError`, it also inherits + from :class:`UnicodeDecodeError` to help the caller catching unicode-related + errors. + """ + + +class PyAsn1UnicodeEncodeError(PyAsn1UnicodeError, UnicodeEncodeError): + """Unicode text encoding error + + The `PyAsn1UnicodeEncodeError` exception represents a failure to + serialize unicode text. + + Apart from inheriting from :class:`PyAsn1UnicodeError`, it also inherits + from :class:`UnicodeEncodeError` to help the caller catching + unicode-related errors. + """ + + diff --git a/server/www/packages/packages-linux/x64/pyasn1/type/base.py b/server/www/packages/packages-linux/x64/pyasn1/type/base.py index adaab22..994f1c9 100644 --- a/server/www/packages/packages-linux/x64/pyasn1/type/base.py +++ b/server/www/packages/packages-linux/x64/pyasn1/type/base.py @@ -1,7 +1,7 @@ # # This file is part of pyasn1 software. # -# Copyright (c) 2005-2018, Ilya Etingof +# Copyright (c) 2005-2019, Ilya Etingof # License: http://snmplabs.com/pyasn1/license.html # import sys @@ -12,7 +12,8 @@ from pyasn1.type import constraint from pyasn1.type import tag from pyasn1.type import tagmap -__all__ = ['Asn1Item', 'Asn1ItemBase', 'AbstractSimpleAsn1Item', 'AbstractConstructedAsn1Item'] +__all__ = ['Asn1Item', 'Asn1Type', 'SimpleAsn1Type', + 'ConstructedAsn1Type'] class Asn1Item(object): @@ -25,7 +26,17 @@ class Asn1Item(object): return Asn1Item._typeCounter -class Asn1ItemBase(Asn1Item): +class Asn1Type(Asn1Item): + """Base class for all classes representing ASN.1 types. + + In the user code, |ASN.1| class is normally used only for telling + ASN.1 objects from others. + + Note + ---- + For as long as ASN.1 is concerned, a way to compare ASN.1 types + is to use :meth:`isSameTypeWith` and :meth:`isSuperTypeOf` methods. + """ #: Set or return a :py:class:`~pyasn1.type.tag.TagSet` object representing #: ASN.1 tag(s) associated with |ASN.1| type. tagSet = tag.TagSet() @@ -91,8 +102,8 @@ class Asn1ItemBase(Asn1Item): Returns ------- : :class:`bool` - :class:`True` if *other* is |ASN.1| type, - :class:`False` otherwise. + :obj:`True` if *other* is |ASN.1| type, + :obj:`False` otherwise. """ return (self is other or (not matchTags or self.tagSet == other.tagSet) and @@ -115,8 +126,8 @@ class Asn1ItemBase(Asn1Item): Returns ------- : :class:`bool` - :class:`True` if *other* is a subtype of |ASN.1| type, - :class:`False` otherwise. + :obj:`True` if *other* is a subtype of |ASN.1| type, + :obj:`False` otherwise. """ return (not matchTags or (self.tagSet.isSuperTagSetOf(other.tagSet)) and @@ -146,9 +157,13 @@ class Asn1ItemBase(Asn1Item): def getSubtypeSpec(self): return self.subtypeSpec + # backward compatibility def hasValue(self): return self.isValue +# Backward compatibility +Asn1ItemBase = Asn1Type + class NoValue(object): """Create a singleton instance of NoValue class. @@ -221,19 +236,31 @@ class NoValue(object): raise error.PyAsn1Error('Attempted "%s" operation on ASN.1 schema object' % attr) def __repr__(self): - return '<%s object at 0x%x>' % (self.__class__.__name__, id(self)) + return '<%s object>' % self.__class__.__name__ noValue = NoValue() -# Base class for "simple" ASN.1 objects. These are immutable. -class AbstractSimpleAsn1Item(Asn1ItemBase): +class SimpleAsn1Type(Asn1Type): + """Base class for all simple classes representing ASN.1 types. + + ASN.1 distinguishes types by their ability to hold other objects. + Scalar types are known as *simple* in ASN.1. + + In the user code, |ASN.1| class is normally used only for telling + ASN.1 objects from others. + + Note + ---- + For as long as ASN.1 is concerned, a way to compare ASN.1 types + is to use :meth:`isSameTypeWith` and :meth:`isSuperTypeOf` methods. + """ #: Default payload value defaultValue = noValue def __init__(self, value=noValue, **kwargs): - Asn1ItemBase.__init__(self, **kwargs) + Asn1Type.__init__(self, **kwargs) if value is noValue: value = self.defaultValue else: @@ -248,19 +275,18 @@ class AbstractSimpleAsn1Item(Asn1ItemBase): self._value = value def __repr__(self): - representation = '%s %s object at 0x%x' % ( - self.__class__.__name__, self.isValue and 'value' or 'schema', id(self) - ) + representation = '%s %s object' % ( + self.__class__.__name__, self.isValue and 'value' or 'schema') for attr, value in self.readOnly.items(): if value: - representation += ' %s %s' % (attr, value) + representation += ', %s %s' % (attr, value) if self.isValue: value = self.prettyPrint() if len(value) > 32: value = value[:16] + '...' + value[-16:] - representation += ' payload [%s]' % value + representation += ', payload [%s]' % value return '<%s>' % representation @@ -296,17 +322,18 @@ class AbstractSimpleAsn1Item(Asn1ItemBase): def isValue(self): """Indicate that |ASN.1| object represents ASN.1 value. - If *isValue* is `False` then this object represents just ASN.1 schema. + If *isValue* is :obj:`False` then this object represents just + ASN.1 schema. - If *isValue* is `True` then, in addition to its ASN.1 schema features, - this object can also be used like a Python built-in object (e.g. `int`, - `str`, `dict` etc.). + If *isValue* is :obj:`True` then, in addition to its ASN.1 schema + features, this object can also be used like a Python built-in object + (e.g. :class:`int`, :class:`str`, :class:`dict` etc.). Returns ------- : :class:`bool` - :class:`False` if object represents just ASN.1 schema. - :class:`True` if object represents ASN.1 schema and can be used as a normal value. + :obj:`False` if object represents just ASN.1 schema. + :obj:`True` if object represents ASN.1 schema and can be used as a normal value. Note ---- @@ -343,10 +370,10 @@ class AbstractSimpleAsn1Item(Asn1ItemBase): value = self._value - initilaizers = self.readOnly.copy() - initilaizers.update(kwargs) + initializers = self.readOnly.copy() + initializers.update(kwargs) - return self.__class__(value, **initilaizers) + return self.__class__(value, **initializers) def subtype(self, value=noValue, **kwargs): """Create a specialization of |ASN.1| schema or value object. @@ -425,10 +452,12 @@ class AbstractSimpleAsn1Item(Asn1ItemBase): def prettyPrint(self, scope=0): return self.prettyOut(self._value) - # noinspection PyUnusedLocal def prettyPrintType(self, scope=0): return '%s -> %s' % (self.tagSet, self.__class__.__name__) +# Backward compatibility +AbstractSimpleAsn1Item = SimpleAsn1Type + # # Constructed types: # * There are five of them: Sequence, SequenceOf/SetOf, Set and Choice @@ -449,67 +478,102 @@ class AbstractSimpleAsn1Item(Asn1ItemBase): # -class AbstractConstructedAsn1Item(Asn1ItemBase): +class ConstructedAsn1Type(Asn1Type): + """Base class for all constructed classes representing ASN.1 types. - #: If `True`, requires exact component type matching, + ASN.1 distinguishes types by their ability to hold other objects. + Those "nesting" types are known as *constructed* in ASN.1. + + In the user code, |ASN.1| class is normally used only for telling + ASN.1 objects from others. + + Note + ---- + For as long as ASN.1 is concerned, a way to compare ASN.1 types + is to use :meth:`isSameTypeWith` and :meth:`isSuperTypeOf` methods. + """ + + #: If :obj:`True`, requires exact component type matching, #: otherwise subtype relation is only enforced strictConstraints = False componentType = None - sizeSpec = None + + # backward compatibility, unused + sizeSpec = constraint.ConstraintsIntersection() def __init__(self, **kwargs): readOnly = { 'componentType': self.componentType, + # backward compatibility, unused 'sizeSpec': self.sizeSpec } + + # backward compatibility: preserve legacy sizeSpec support + kwargs = self._moveSizeSpec(**kwargs) + readOnly.update(kwargs) - Asn1ItemBase.__init__(self, **readOnly) + Asn1Type.__init__(self, **readOnly) - self._componentValues = [] + def _moveSizeSpec(self, **kwargs): + # backward compatibility, unused + sizeSpec = kwargs.pop('sizeSpec', self.sizeSpec) + if sizeSpec: + subtypeSpec = kwargs.pop('subtypeSpec', self.subtypeSpec) + if subtypeSpec: + subtypeSpec = sizeSpec + + else: + subtypeSpec += sizeSpec + + kwargs['subtypeSpec'] = subtypeSpec + + return kwargs def __repr__(self): - representation = '%s %s object at 0x%x' % ( - self.__class__.__name__, self.isValue and 'value' or 'schema', id(self) + representation = '%s %s object' % ( + self.__class__.__name__, self.isValue and 'value' or 'schema' ) for attr, value in self.readOnly.items(): if value is not noValue: - representation += ' %s=%r' % (attr, value) + representation += ', %s=%r' % (attr, value) - if self.isValue and self._componentValues: - representation += ' payload [%s]' % ', '.join([repr(x) for x in self._componentValues]) + if self.isValue and self.components: + representation += ', payload [%s]' % ', '.join( + [repr(x) for x in self.components]) return '<%s>' % representation def __eq__(self, other): - return self is other and True or self._componentValues == other + return self is other or self.components == other def __ne__(self, other): - return self._componentValues != other + return self.components != other def __lt__(self, other): - return self._componentValues < other + return self.components < other def __le__(self, other): - return self._componentValues <= other + return self.components <= other def __gt__(self, other): - return self._componentValues > other + return self.components > other def __ge__(self, other): - return self._componentValues >= other + return self.components >= other if sys.version_info[0] <= 2: def __nonzero__(self): - return self._componentValues and True or False + return bool(self.components) else: def __bool__(self): - return self._componentValues and True or False + return bool(self.components) - def __len__(self): - return len(self._componentValues) + @property + def components(self): + raise error.PyAsn1Error('Method not implemented') def _cloneComponentValues(self, myClone, cloneValueFlag): pass @@ -535,15 +599,14 @@ class AbstractConstructedAsn1Item(Asn1ItemBase): Note ---- Due to the mutable nature of the |ASN.1| object, even if no arguments - are supplied, new |ASN.1| object will always be created as a shallow - copy of `self`. + are supplied, a new |ASN.1| object will be created and returned. """ cloneValueFlag = kwargs.pop('cloneValueFlag', False) - initilaizers = self.readOnly.copy() - initilaizers.update(kwargs) + initializers = self.readOnly.copy() + initializers.update(kwargs) - clone = self.__class__(**initilaizers) + clone = self.__class__(**initializers) if cloneValueFlag: self._cloneComponentValues(clone, cloneValueFlag) @@ -588,9 +651,8 @@ class AbstractConstructedAsn1Item(Asn1ItemBase): Note ---- - Due to the immutable nature of the |ASN.1| object, if no arguments - are supplied, no new |ASN.1| object will be created and `self` will - be returned instead. + Due to the mutable nature of the |ASN.1| object, even if no arguments + are supplied, a new |ASN.1| object will be created and returned. """ initializers = self.readOnly.copy() @@ -615,9 +677,6 @@ class AbstractConstructedAsn1Item(Asn1ItemBase): return clone - def verifySizeSpec(self): - self.sizeSpec(self) - def getComponentByPosition(self, idx): raise error.PyAsn1Error('Method not implemented') @@ -631,9 +690,6 @@ class AbstractConstructedAsn1Item(Asn1ItemBase): self[k] = kwargs[k] return self - def clear(self): - self._componentValues = [] - # backward compatibility def setDefaultComponents(self): @@ -641,3 +697,11 @@ class AbstractConstructedAsn1Item(Asn1ItemBase): def getComponentType(self): return self.componentType + + # backward compatibility, unused + def verifySizeSpec(self): + self.subtypeSpec(self) + + + # Backward compatibility +AbstractConstructedAsn1Item = ConstructedAsn1Type diff --git a/server/www/packages/packages-linux/x64/pyasn1/type/char.py b/server/www/packages/packages-linux/x64/pyasn1/type/char.py index 493badb..06074da 100644 --- a/server/www/packages/packages-linux/x64/pyasn1/type/char.py +++ b/server/www/packages/packages-linux/x64/pyasn1/type/char.py @@ -1,7 +1,7 @@ # # This file is part of pyasn1 software. # -# Copyright (c) 2005-2018, Ilya Etingof +# Copyright (c) 2005-2019, Ilya Etingof # License: http://snmplabs.com/pyasn1/license.html # import sys @@ -21,21 +21,27 @@ noValue = univ.noValue class AbstractCharacterString(univ.OctetString): """Creates |ASN.1| schema or value object. - |ASN.1| objects are immutable and duck-type Python 2 :class:`unicode` or Python 3 :class:`str`. - When used in octet-stream context, |ASN.1| type assumes "|encoding|" encoding. + |ASN.1| class is based on :class:`~pyasn1.type.base.SimpleAsn1Type`, + its objects are immutable and duck-type Python 2 :class:`str` or Python 3 + :class:`bytes`. When used in octet-stream context, |ASN.1| type assumes + "|encoding|" encoding. Keyword Args ------------ value: :class:`unicode`, :class:`str`, :class:`bytes` or |ASN.1| object - unicode object (Python 2) or string (Python 3), alternatively string - (Python 2) or bytes (Python 3) representing octet-stream of serialised - unicode string (note `encoding` parameter) or |ASN.1| class instance. + :class:`unicode` object (Python 2) or :class:`str` (Python 3), + alternatively :class:`str` (Python 2) or :class:`bytes` (Python 3) + representing octet-stream of serialised unicode string + (note `encoding` parameter) or |ASN.1| class instance. + If `value` is not given, schema object will be created. tagSet: :py:class:`~pyasn1.type.tag.TagSet` Object representing non-default ASN.1 tag(s) subtypeSpec: :py:class:`~pyasn1.type.constraint.ConstraintsIntersection` - Object representing non-default ASN.1 subtype constraint(s) + Object representing non-default ASN.1 subtype constraint(s). Constraints + verification for |ASN.1| type occurs automatically on object + instantiation. encoding: :py:class:`str` Unicode codec ID to encode/decode :class:`unicode` (Python 2) or @@ -44,7 +50,7 @@ class AbstractCharacterString(univ.OctetString): Raises ------ - :py:class:`~pyasn1.error.PyAsn1Error` + ~pyasn1.error.ValueConstraintError, ~pyasn1.error.PyAsn1Error On constraint violation or bad initializer. """ @@ -55,8 +61,10 @@ class AbstractCharacterString(univ.OctetString): return self._value.encode(self.encoding) except UnicodeEncodeError: - raise error.PyAsn1Error( - "Can't encode string '%s' with codec %s" % (self._value, self.encoding) + exc = sys.exc_info()[1] + raise error.PyAsn1UnicodeEncodeError( + "Can't encode string '%s' with codec " + "%s" % (self._value, self.encoding), exc ) def __unicode__(self): @@ -76,8 +84,10 @@ class AbstractCharacterString(univ.OctetString): return unicode(value) except (UnicodeDecodeError, LookupError): - raise error.PyAsn1Error( - "Can't decode string '%s' with codec %s" % (value, self.encoding) + exc = sys.exc_info()[1] + raise error.PyAsn1UnicodeDecodeError( + "Can't decode string '%s' with codec " + "%s" % (value, self.encoding), exc ) def asOctets(self, padding=True): @@ -95,8 +105,10 @@ class AbstractCharacterString(univ.OctetString): try: return self._value.encode(self.encoding) except UnicodeEncodeError: - raise error.PyAsn1Error( - "Can't encode string '%s' with codec %s" % (self._value, self.encoding) + exc = sys.exc_info()[1] + raise error.PyAsn1UnicodeEncodeError( + "Can't encode string '%s' with codec " + "%s" % (self._value, self.encoding), exc ) def prettyIn(self, value): @@ -113,8 +125,10 @@ class AbstractCharacterString(univ.OctetString): return str(value) except (UnicodeDecodeError, LookupError): - raise error.PyAsn1Error( - "Can't decode string '%s' with codec %s" % (value, self.encoding) + exc = sys.exc_info()[1] + raise error.PyAsn1UnicodeDecodeError( + "Can't decode string '%s' with codec " + "%s" % (value, self.encoding), exc ) def asOctets(self, padding=True): diff --git a/server/www/packages/packages-linux/x64/pyasn1/type/constraint.py b/server/www/packages/packages-linux/x64/pyasn1/type/constraint.py index a704331..8f152e9 100644 --- a/server/www/packages/packages-linux/x64/pyasn1/type/constraint.py +++ b/server/www/packages/packages-linux/x64/pyasn1/type/constraint.py @@ -1,7 +1,7 @@ # # This file is part of pyasn1 software. # -# Copyright (c) 2005-2018, Ilya Etingof +# Copyright (c) 2005-2019, Ilya Etingof # License: http://snmplabs.com/pyasn1/license.html # # Original concept and code by Mike C. Fletcher. @@ -37,10 +37,11 @@ class AbstractConstraint(object): ) def __repr__(self): - representation = '%s object at 0x%x' % (self.__class__.__name__, id(self)) + representation = '%s object' % (self.__class__.__name__) if self._values: - representation += ' consts %s' % ', '.join([repr(x) for x in self._values]) + representation += ', consts %s' % ', '.join( + [repr(x) for x in self._values]) return '<%s>' % representation @@ -102,12 +103,17 @@ class SingleValueConstraint(AbstractConstraint): The SingleValueConstraint satisfies any value that is present in the set of permitted values. + Objects of this type are iterable (emitting constraint values) and + can act as operands for some arithmetic operations e.g. addition + and subtraction. The latter can be used for combining multiple + SingleValueConstraint objects into one. + The SingleValueConstraint object can be applied to any ASN.1 type. Parameters ---------- - \*values: :class:`int` + *values: :class:`int` Full set of values permitted by this constraint object. Examples @@ -136,6 +142,23 @@ class SingleValueConstraint(AbstractConstraint): if value not in self._set: raise error.ValueConstraintError(value) + # Constrains can be merged or reduced + + def __contains__(self, item): + return item in self._set + + def __iter__(self): + return iter(self._set) + + def __sub__(self, constraint): + return self.__class__(*(self._set.difference(constraint))) + + def __add__(self, constraint): + return self.__class__(*(self._set.union(constraint))) + + def __sub__(self, constraint): + return self.__class__(*(self._set.difference(constraint))) + class ContainedSubtypeConstraint(AbstractConstraint): """Create a ContainedSubtypeConstraint object. @@ -149,7 +172,7 @@ class ContainedSubtypeConstraint(AbstractConstraint): Parameters ---------- - \*values: + *values: Full set of values and constraint objects permitted by this constraint object. @@ -304,17 +327,21 @@ class PermittedAlphabetConstraint(SingleValueConstraint): string for as long as all its characters are present in the set of permitted characters. + Objects of this type are iterable (emitting constraint values) and + can act as operands for some arithmetic operations e.g. addition + and subtraction. + The PermittedAlphabetConstraint object can only be applied to the :ref:`character ASN.1 types ` such as :class:`~pyasn1.type.char.IA5String`. Parameters ---------- - \*alphabet: :class:`str` + *alphabet: :class:`str` Full set of characters permitted by this constraint object. - Examples - -------- + Example + ------- .. code-block:: python class BooleanValue(IA5String): @@ -331,6 +358,42 @@ class PermittedAlphabetConstraint(SingleValueConstraint): # this will raise ValueConstraintError garbage = BooleanValue('TAF') + + ASN.1 `FROM ... EXCEPT ...` clause can be modelled by combining multiple + PermittedAlphabetConstraint objects into one: + + Example + ------- + .. code-block:: python + + class Lipogramme(IA5String): + ''' + ASN.1 specification: + + Lipogramme ::= + IA5String (FROM (ALL EXCEPT ("e"|"E"))) + ''' + subtypeSpec = ( + PermittedAlphabetConstraint(*string.printable) - + PermittedAlphabetConstraint('e', 'E') + ) + + # this will succeed + lipogramme = Lipogramme('A work of fiction?') + + # this will raise ValueConstraintError + lipogramme = Lipogramme('Eel') + + Note + ---- + Although `ConstraintsExclusion` object could seemingly be used for this + purpose, practically, for it to work, it needs to represent its operand + constraints as sets and intersect one with the other. That would require + the insight into the constraint values (and their types) that are otherwise + hidden inside the constraint object. + + Therefore it's more practical to model `EXCEPT` clause at + `PermittedAlphabetConstraint` level instead. """ def _setValues(self, values): self._values = values @@ -341,6 +404,151 @@ class PermittedAlphabetConstraint(SingleValueConstraint): raise error.ValueConstraintError(value) +class ComponentPresentConstraint(AbstractConstraint): + """Create a ComponentPresentConstraint object. + + The ComponentPresentConstraint is only satisfied when the value + is not `None`. + + The ComponentPresentConstraint object is typically used with + `WithComponentsConstraint`. + + Examples + -------- + .. code-block:: python + + present = ComponentPresentConstraint() + + # this will succeed + present('whatever') + + # this will raise ValueConstraintError + present(None) + """ + def _setValues(self, values): + self._values = ('',) + + if values: + raise error.PyAsn1Error('No arguments expected') + + def _testValue(self, value, idx): + if value is None: + raise error.ValueConstraintError( + 'Component is not present:') + + +class ComponentAbsentConstraint(AbstractConstraint): + """Create a ComponentAbsentConstraint object. + + The ComponentAbsentConstraint is only satisfied when the value + is `None`. + + The ComponentAbsentConstraint object is typically used with + `WithComponentsConstraint`. + + Examples + -------- + .. code-block:: python + + absent = ComponentAbsentConstraint() + + # this will succeed + absent(None) + + # this will raise ValueConstraintError + absent('whatever') + """ + def _setValues(self, values): + self._values = ('',) + + if values: + raise error.PyAsn1Error('No arguments expected') + + def _testValue(self, value, idx): + if value is not None: + raise error.ValueConstraintError( + 'Component is not absent: %r' % value) + + +class WithComponentsConstraint(AbstractConstraint): + """Create a WithComponentsConstraint object. + + The `WithComponentsConstraint` satisfies any mapping object that has + constrained fields present or absent, what is indicated by + `ComponentPresentConstraint` and `ComponentAbsentConstraint` + objects respectively. + + The `WithComponentsConstraint` object is typically applied + to :class:`~pyasn1.type.univ.Set` or + :class:`~pyasn1.type.univ.Sequence` types. + + Parameters + ---------- + *fields: :class:`tuple` + Zero or more tuples of (`field`, `constraint`) indicating constrained + fields. + + Notes + ----- + On top of the primary use of `WithComponentsConstraint` (ensuring presence + or absence of particular components of a :class:`~pyasn1.type.univ.Set` or + :class:`~pyasn1.type.univ.Sequence`), it is also possible to pass any other + constraint objects or their combinations. In case of scalar fields, these + constraints will be verified in addition to the constraints belonging to + scalar components themselves. However, formally, these additional + constraints do not change the type of these ASN.1 objects. + + Examples + -------- + + .. code-block:: python + + class Item(Sequence): # Set is similar + ''' + ASN.1 specification: + + Item ::= SEQUENCE { + id INTEGER OPTIONAL, + name OCTET STRING OPTIONAL + } WITH COMPONENTS id PRESENT, name ABSENT | id ABSENT, name PRESENT + ''' + componentType = NamedTypes( + OptionalNamedType('id', Integer()), + OptionalNamedType('name', OctetString()) + ) + withComponents = ConstraintsUnion( + WithComponentsConstraint( + ('id', ComponentPresentConstraint()), + ('name', ComponentAbsentConstraint()) + ), + WithComponentsConstraint( + ('id', ComponentAbsentConstraint()), + ('name', ComponentPresentConstraint()) + ) + ) + + item = Item() + + # This will succeed + item['id'] = 1 + + # This will succeed + item.reset() + item['name'] = 'John' + + # This will fail (on encoding) + item.reset() + descr['id'] = 1 + descr['name'] = 'John' + """ + def _testValue(self, value, idx): + for field, constraint in self._values: + constraint(value.get(field)) + + def _setValues(self, values): + AbstractConstraint._setValues(self, values) + + # This is a bit kludgy, meaning two op modes within a single constraint class InnerTypeConstraint(AbstractConstraint): """Value must satisfy the type and presence constraints""" @@ -352,7 +560,7 @@ class InnerTypeConstraint(AbstractConstraint): if idx not in self.__multipleTypeConstraint: raise error.ValueConstraintError(value) constraint, status = self.__multipleTypeConstraint[idx] - if status == 'ABSENT': # XXX presense is not checked! + if status == 'ABSENT': # XXX presence is not checked! raise error.ValueConstraintError(value) constraint(value) @@ -380,49 +588,41 @@ class ConstraintsExclusion(AbstractConstraint): Parameters ---------- - constraint: - Constraint or logic operator object. + *constraints: + Constraint or logic operator objects. Examples -------- .. code-block:: python - class Lipogramme(IA5STRING): - ''' - ASN.1 specification: - - Lipogramme ::= - IA5String (FROM (ALL EXCEPT ("e"|"E"))) - ''' + class LuckyNumber(Integer): subtypeSpec = ConstraintsExclusion( - PermittedAlphabetConstraint('e', 'E') + SingleValueConstraint(13) ) # this will succeed - lipogramme = Lipogramme('A work of fiction?') + luckyNumber = LuckyNumber(12) # this will raise ValueConstraintError - lipogramme = Lipogramme('Eel') + luckyNumber = LuckyNumber(13) - Warning - ------- - The above example involving PermittedAlphabetConstraint might - not work due to the way how PermittedAlphabetConstraint works. - The other constraints might work with ConstraintsExclusion - though. + Note + ---- + The `FROM ... EXCEPT ...` ASN.1 clause should be modeled by combining + constraint objects into one. See `PermittedAlphabetConstraint` for more + information. """ def _testValue(self, value, idx): - try: - self._values[0](value, idx) - except error.ValueConstraintError: - return - else: + for constraint in self._values: + try: + constraint(value, idx) + + except error.ValueConstraintError: + continue + raise error.ValueConstraintError(value) def _setValues(self, values): - if len(values) != 1: - raise error.PyAsn1Error('Single constraint expected') - AbstractConstraint._setValues(self, values) @@ -467,7 +667,7 @@ class ConstraintsIntersection(AbstractConstraintSet): Parameters ---------- - \*constraints: + *constraints: Constraint or logic operator objects. Examples @@ -500,8 +700,8 @@ class ConstraintsIntersection(AbstractConstraintSet): class ConstraintsUnion(AbstractConstraintSet): """Create a ConstraintsUnion logic operator object. - The ConstraintsUnion logic operator only succeeds if - *at least a single* operand succeeds. + The ConstraintsUnion logic operator succeeds if + *at least* a single operand succeeds. The ConstraintsUnion object can be applied to any constraint and logic operator objects. @@ -511,7 +711,7 @@ class ConstraintsUnion(AbstractConstraintSet): Parameters ---------- - \*constraints: + *constraints: Constraint or logic operator objects. Examples @@ -525,7 +725,7 @@ class ConstraintsUnion(AbstractConstraintSet): CapitalOrSmall ::= IA5String (FROM ("A".."Z") | FROM ("a".."z")) ''' - subtypeSpec = ConstraintsIntersection( + subtypeSpec = ConstraintsUnion( PermittedAlphabetConstraint('A', 'Z'), PermittedAlphabetConstraint('a', 'z') ) diff --git a/server/www/packages/packages-linux/x64/pyasn1/type/error.py b/server/www/packages/packages-linux/x64/pyasn1/type/error.py index b2056bd..80fcf3b 100644 --- a/server/www/packages/packages-linux/x64/pyasn1/type/error.py +++ b/server/www/packages/packages-linux/x64/pyasn1/type/error.py @@ -1,7 +1,7 @@ # # This file is part of pyasn1 software. # -# Copyright (c) 2005-2018, Ilya Etingof +# Copyright (c) 2005-2019, Ilya Etingof # License: http://snmplabs.com/pyasn1/license.html # from pyasn1.error import PyAsn1Error diff --git a/server/www/packages/packages-linux/x64/pyasn1/type/namedtype.py b/server/www/packages/packages-linux/x64/pyasn1/type/namedtype.py index f162d19..cbc1429 100644 --- a/server/www/packages/packages-linux/x64/pyasn1/type/namedtype.py +++ b/server/www/packages/packages-linux/x64/pyasn1/type/namedtype.py @@ -1,7 +1,7 @@ # # This file is part of pyasn1 software. # -# Copyright (c) 2005-2018, Ilya Etingof +# Copyright (c) 2005-2019, Ilya Etingof # License: http://snmplabs.com/pyasn1/license.html # import sys @@ -49,9 +49,10 @@ class NamedType(object): representation = '%s=%r' % (self.name, self.asn1Object) if self.openType: - representation += ' openType: %r' % self.openType + representation += ', open type %r' % self.openType - return '<%s object at 0x%x type %s>' % (self.__class__.__name__, id(self), representation) + return '<%s object, type %s>' % ( + self.__class__.__name__, representation) def __eq__(self, other): return self.__nameAndType == other @@ -173,7 +174,8 @@ class NamedTypes(object): def __repr__(self): representation = ', '.join(['%r' % x for x in self.__namedTypes]) - return '<%s object at 0x%x types %s>' % (self.__class__.__name__, id(self), representation) + return '<%s object, types %s>' % ( + self.__class__.__name__, representation) def __eq__(self, other): return self.__namedTypes == other @@ -265,18 +267,18 @@ class NamedTypes(object): return nameToPosMap def __computeAmbiguousTypes(self): - ambigiousTypes = {} - partialAmbigiousTypes = () + ambiguousTypes = {} + partialAmbiguousTypes = () for idx, namedType in reversed(tuple(enumerate(self.__namedTypes))): if namedType.isOptional or namedType.isDefaulted: - partialAmbigiousTypes = (namedType,) + partialAmbigiousTypes + partialAmbiguousTypes = (namedType,) + partialAmbiguousTypes else: - partialAmbigiousTypes = (namedType,) - if len(partialAmbigiousTypes) == len(self.__namedTypes): - ambigiousTypes[idx] = self + partialAmbiguousTypes = (namedType,) + if len(partialAmbiguousTypes) == len(self.__namedTypes): + ambiguousTypes[idx] = self else: - ambigiousTypes[idx] = NamedTypes(*partialAmbigiousTypes, **dict(terminal=True)) - return ambigiousTypes + ambiguousTypes[idx] = NamedTypes(*partialAmbiguousTypes, **dict(terminal=True)) + return ambiguousTypes def getTypeByPosition(self, idx): """Return ASN.1 type object by its position in fields set. @@ -293,7 +295,7 @@ class NamedTypes(object): Raises ------ - : :class:`~pyasn1.error.PyAsn1Error` + ~pyasn1.error.PyAsn1Error If given position is out of fields range """ try: @@ -317,7 +319,7 @@ class NamedTypes(object): Raises ------ - : :class:`~pyasn1.error.PyAsn1Error` + ~pyasn1.error.PyAsn1Error If *tagSet* is not present or ASN.1 types are not unique within callee *NamedTypes* """ try: @@ -341,7 +343,7 @@ class NamedTypes(object): Raises ------ - : :class:`~pyasn1.error.PyAsn1Error` + ~pyasn1.error.PyAsn1Error If given field name is not present in callee *NamedTypes* """ try: @@ -365,7 +367,7 @@ class NamedTypes(object): Raises ------ - : :class:`~pyasn1.error.PyAsn1Error` + ~pyasn1.error.PyAsn1Error If *name* is not present or not unique within callee *NamedTypes* """ try: @@ -394,7 +396,7 @@ class NamedTypes(object): Raises ------ - : :class:`~pyasn1.error.PyAsn1Error` + ~pyasn1.error.PyAsn1Error If given position is out of fields range """ try: @@ -426,7 +428,7 @@ class NamedTypes(object): Raises ------ - : :class:`~pyasn1.error.PyAsn1Error` + ~pyasn1.error.PyAsn1Error If *tagSet* is not present or not unique within callee *NamedTypes* or *idx* is out of fields range """ diff --git a/server/www/packages/packages-linux/x64/pyasn1/type/namedval.py b/server/www/packages/packages-linux/x64/pyasn1/type/namedval.py index 59257e4..4247597 100644 --- a/server/www/packages/packages-linux/x64/pyasn1/type/namedval.py +++ b/server/www/packages/packages-linux/x64/pyasn1/type/namedval.py @@ -1,7 +1,7 @@ # # This file is part of pyasn1 software. # -# Copyright (c) 2005-2018, Ilya Etingof +# Copyright (c) 2005-2019, Ilya Etingof # License: http://snmplabs.com/pyasn1/license.html # # ASN.1 named integers @@ -23,7 +23,7 @@ class NamedValues(object): Parameters ---------- - \*args: variable number of two-element :py:class:`tuple` + *args: variable number of two-element :py:class:`tuple` name: :py:class:`str` Value label @@ -109,7 +109,8 @@ class NamedValues(object): if len(representation) > 64: representation = representation[:32] + '...' + representation[-32:] - return '<%s object 0x%x enums %s>' % (self.__class__.__name__, id(self), representation) + return '<%s object, enums %s>' % ( + self.__class__.__name__, representation) def __eq__(self, other): return dict(self) == other diff --git a/server/www/packages/packages-linux/x64/pyasn1/type/opentype.py b/server/www/packages/packages-linux/x64/pyasn1/type/opentype.py index d14ab34..29645f0 100644 --- a/server/www/packages/packages-linux/x64/pyasn1/type/opentype.py +++ b/server/www/packages/packages-linux/x64/pyasn1/type/opentype.py @@ -1,7 +1,7 @@ # # This file is part of pyasn1 software. # -# Copyright (c) 2005-2018, Ilya Etingof +# Copyright (c) 2005-2019, Ilya Etingof # License: http://snmplabs.com/pyasn1/license.html # @@ -11,11 +11,22 @@ __all__ = ['OpenType'] class OpenType(object): """Create ASN.1 type map indexed by a value - The *DefinedBy* object models the ASN.1 *DEFINED BY* clause which maps - values to ASN.1 types in the context of the ASN.1 SEQUENCE/SET type. + The *OpenType* object models an untyped field of a constructed ASN.1 + type. In ASN.1 syntax it is usually represented by the + `ANY DEFINED BY` for scalars or `SET OF ANY DEFINED BY`, + `SEQUENCE OF ANY DEFINED BY` for container types clauses. Typically + used together with :class:`~pyasn1.type.univ.Any` object. - OpenType objects are duck-type a read-only Python :class:`dict` objects, - however the passed `typeMap` is stored by reference. + OpenType objects duck-type a read-only Python :class:`dict` objects, + however the passed `typeMap` is not copied, but stored by reference. + That means the user can manipulate `typeMap` at run time having this + reflected on *OpenType* object behavior. + + The |OpenType| class models an untyped field of a constructed ASN.1 + type. In ASN.1 syntax it is usually represented by the + `ANY DEFINED BY` for scalars or `SET OF ANY DEFINED BY`, + `SEQUENCE OF ANY DEFINED BY` for container types clauses. Typically + used with :class:`~pyasn1.type.univ.Any` type. Parameters ---------- @@ -28,12 +39,14 @@ class OpenType(object): Examples -------- + + For untyped scalars: + .. code-block:: python openType = OpenType( - 'id', - {1: Integer(), - 2: OctetString()} + 'id', {1: Integer(), + 2: OctetString()} ) Sequence( componentType=NamedTypes( @@ -41,6 +54,22 @@ class OpenType(object): NamedType('blob', Any(), openType=openType) ) ) + + For untyped `SET OF` or `SEQUENCE OF` vectors: + + .. code-block:: python + + openType = OpenType( + 'id', {1: Integer(), + 2: OctetString()} + ) + Sequence( + componentType=NamedTypes( + NamedType('id', Integer()), + NamedType('blob', SetOf(componentType=Any()), + openType=openType) + ) + ) """ def __init__(self, name, typeMap=None): diff --git a/server/www/packages/packages-linux/x64/pyasn1/type/tag.py b/server/www/packages/packages-linux/x64/pyasn1/type/tag.py index 95c226f..b88a734 100644 --- a/server/www/packages/packages-linux/x64/pyasn1/type/tag.py +++ b/server/www/packages/packages-linux/x64/pyasn1/type/tag.py @@ -1,7 +1,7 @@ # # This file is part of pyasn1 software. # -# Copyright (c) 2005-2018, Ilya Etingof +# Copyright (c) 2005-2019, Ilya Etingof # License: http://snmplabs.com/pyasn1/license.html # from pyasn1 import error @@ -64,8 +64,10 @@ class Tag(object): self.__hash = hash(self.__tagClassId) def __repr__(self): - representation = '[%s:%s:%s]' % (self.__tagClass, self.__tagFormat, self.__tagId) - return '<%s object at 0x%x tag %s>' % (self.__class__.__name__, id(self), representation) + representation = '[%s:%s:%s]' % ( + self.__tagClass, self.__tagFormat, self.__tagId) + return '<%s object, tag %s>' % ( + self.__class__.__name__, representation) def __eq__(self, other): return self.__tagClassId == other @@ -199,7 +201,7 @@ class TagSet(object): else: representation = 'untagged' - return '<%s object at 0x%x %s>' % (self.__class__.__name__, id(self), representation) + return '<%s object, %s>' % (self.__class__.__name__, representation) def __add__(self, superTag): return self.__class__(self.__baseTag, *self.__superTags + (superTag,)) @@ -318,7 +320,7 @@ class TagSet(object): Returns ------- : :py:class:`bool` - `True` if callee is a supertype of *tagSet* + :obj:`True` if callee is a supertype of *tagSet* """ if len(tagSet) < self.__lenOfSuperTags: return False diff --git a/server/www/packages/packages-linux/x64/pyasn1/type/tagmap.py b/server/www/packages/packages-linux/x64/pyasn1/type/tagmap.py index a9d237f..6f5163b 100644 --- a/server/www/packages/packages-linux/x64/pyasn1/type/tagmap.py +++ b/server/www/packages/packages-linux/x64/pyasn1/type/tagmap.py @@ -1,7 +1,7 @@ # # This file is part of pyasn1 software. # -# Copyright (c) 2005-2018, Ilya Etingof +# Copyright (c) 2005-2019, Ilya Etingof # License: http://snmplabs.com/pyasn1/license.html # from pyasn1 import error @@ -56,16 +56,16 @@ class TagMap(object): return iter(self.__presentTypes) def __repr__(self): - representation = '%s object at 0x%x' % (self.__class__.__name__, id(self)) + representation = '%s object' % self.__class__.__name__ if self.__presentTypes: - representation += ' present %s' % repr(self.__presentTypes) + representation += ', present %s' % repr(self.__presentTypes) if self.__skipTypes: - representation += ' skip %s' % repr(self.__skipTypes) + representation += ', skip %s' % repr(self.__skipTypes) if self.__defaultType is not None: - representation += ' default %s' % repr(self.__defaultType) + representation += ', default %s' % repr(self.__defaultType) return '<%s>' % representation diff --git a/server/www/packages/packages-linux/x64/pyasn1/type/univ.py b/server/www/packages/packages-linux/x64/pyasn1/type/univ.py index 898cf25..aa688b2 100644 --- a/server/www/packages/packages-linux/x64/pyasn1/type/univ.py +++ b/server/www/packages/packages-linux/x64/pyasn1/type/univ.py @@ -1,7 +1,7 @@ # # This file is part of pyasn1 software. # -# Copyright (c) 2005-2018, Ilya Etingof +# Copyright (c) 2005-2019, Ilya Etingof # License: http://snmplabs.com/pyasn1/license.html # import math @@ -31,28 +31,32 @@ __all__ = ['Integer', 'Boolean', 'BitString', 'OctetString', 'Null', # "Simple" ASN.1 types (yet incomplete) -class Integer(base.AbstractSimpleAsn1Item): - """Create |ASN.1| type or object. +class Integer(base.SimpleAsn1Type): + """Create |ASN.1| schema or value object. - |ASN.1| objects are immutable and duck-type Python :class:`int` objects. + |ASN.1| class is based on :class:`~pyasn1.type.base.SimpleAsn1Type`, its + objects are immutable and duck-type Python :class:`int` objects. Keyword Args ------------ value: :class:`int`, :class:`str` or |ASN.1| object - Python integer or string literal or |ASN.1| class instance. + Python :class:`int` or :class:`str` literal or |ASN.1| class + instance. If `value` is not given, schema object will be created. tagSet: :py:class:`~pyasn1.type.tag.TagSet` Object representing non-default ASN.1 tag(s) subtypeSpec: :py:class:`~pyasn1.type.constraint.ConstraintsIntersection` - Object representing non-default ASN.1 subtype constraint(s) + Object representing non-default ASN.1 subtype constraint(s). Constraints + verification for |ASN.1| type occurs automatically on object + instantiation. namedValues: :py:class:`~pyasn1.type.namedval.NamedValues` Object representing non-default symbolic aliases for numbers Raises ------ - :py:class:`~pyasn1.error.PyAsn1Error` + ~pyasn1.error.ValueConstraintError, ~pyasn1.error.PyAsn1Error On constraint violation or bad initializer. Examples @@ -94,13 +98,13 @@ class Integer(base.AbstractSimpleAsn1Item): namedValues = namedval.NamedValues() # Optimization for faster codec lookup - typeId = base.AbstractSimpleAsn1Item.getTypeId() + typeId = base.SimpleAsn1Type.getTypeId() def __init__(self, value=noValue, **kwargs): if 'namedValues' not in kwargs: kwargs['namedValues'] = self.namedValues - base.AbstractSimpleAsn1Item.__init__(self, value, **kwargs) + base.SimpleAsn1Type.__init__(self, value, **kwargs) def __and__(self, value): return self.clone(self._value & value) @@ -187,7 +191,7 @@ class Integer(base.AbstractSimpleAsn1Item): def __rdivmod__(self, value): return self.clone(divmod(value, self._value)) - __hash__ = base.AbstractSimpleAsn1Item.__hash__ + __hash__ = base.SimpleAsn1Type.__hash__ def __int__(self): return int(self._value) @@ -276,27 +280,31 @@ class Integer(base.AbstractSimpleAsn1Item): class Boolean(Integer): - """Create |ASN.1| type or object. + """Create |ASN.1| schema or value object. - |ASN.1| objects are immutable and duck-type Python :class:`int` objects. + |ASN.1| class is based on :class:`~pyasn1.type.base.SimpleAsn1Type`, its + objects are immutable and duck-type Python :class:`int` objects. Keyword Args ------------ value: :class:`int`, :class:`str` or |ASN.1| object - Python integer or boolean or string literal or |ASN.1| class instance. + Python :class:`int` or :class:`str` literal or |ASN.1| class + instance. If `value` is not given, schema object will be created. tagSet: :py:class:`~pyasn1.type.tag.TagSet` Object representing non-default ASN.1 tag(s) subtypeSpec: :py:class:`~pyasn1.type.constraint.ConstraintsIntersection` - Object representing non-default ASN.1 subtype constraint(s) + Object representing non-default ASN.1 subtype constraint(s).Constraints + verification for |ASN.1| type occurs automatically on object + instantiation. namedValues: :py:class:`~pyasn1.type.namedval.NamedValues` Object representing non-default symbolic aliases for numbers Raises ------ - :py:class:`~pyasn1.error.PyAsn1Error` + ~pyasn1.error.ValueConstraintError, ~pyasn1.error.PyAsn1Error On constraint violation or bad initializer. Examples @@ -355,23 +363,27 @@ class SizedInteger(SizedIntegerBase): return self.bitLength -class BitString(base.AbstractSimpleAsn1Item): +class BitString(base.SimpleAsn1Type): """Create |ASN.1| schema or value object. - |ASN.1| objects are immutable and duck-type both Python :class:`tuple` (as a tuple + |ASN.1| class is based on :class:`~pyasn1.type.base.SimpleAsn1Type`, its + objects are immutable and duck-type both Python :class:`tuple` (as a tuple of bits) and :class:`int` objects. Keyword Args ------------ value: :class:`int`, :class:`str` or |ASN.1| object - Python integer or string literal representing binary or hexadecimal - number or sequence of integer bits or |ASN.1| object. + Python :class:`int` or :class:`str` literal representing binary + or hexadecimal number or sequence of integer bits or |ASN.1| object. + If `value` is not given, schema object will be created. tagSet: :py:class:`~pyasn1.type.tag.TagSet` Object representing non-default ASN.1 tag(s) subtypeSpec: :py:class:`~pyasn1.type.constraint.ConstraintsIntersection` - Object representing non-default ASN.1 subtype constraint(s) + Object representing non-default ASN.1 subtype constraint(s). Constraints + verification for |ASN.1| type occurs automatically on object + instantiation. namedValues: :py:class:`~pyasn1.type.namedval.NamedValues` Object representing non-default symbolic aliases for numbers @@ -386,7 +398,7 @@ class BitString(base.AbstractSimpleAsn1Item): Raises ------ - :py:class:`~pyasn1.error.PyAsn1Error` + ~pyasn1.error.ValueConstraintError, ~pyasn1.error.PyAsn1Error On constraint violation or bad initializer. Examples @@ -432,7 +444,7 @@ class BitString(base.AbstractSimpleAsn1Item): namedValues = namedval.NamedValues() # Optimization for faster codec lookup - typeId = base.AbstractSimpleAsn1Item.getTypeId() + typeId = base.SimpleAsn1Type.getTypeId() defaultBinValue = defaultHexValue = noValue @@ -461,7 +473,7 @@ class BitString(base.AbstractSimpleAsn1Item): if 'namedValues' not in kwargs: kwargs['namedValues'] = self.namedValues - base.AbstractSimpleAsn1Item.__init__(self, value, **kwargs) + base.SimpleAsn1Type.__init__(self, value, **kwargs) def __str__(self): return self.asBinary() @@ -720,24 +732,30 @@ except NameError: # Python 2.4 return True -class OctetString(base.AbstractSimpleAsn1Item): +class OctetString(base.SimpleAsn1Type): """Create |ASN.1| schema or value object. - |ASN.1| objects are immutable and duck-type Python 2 :class:`str` or Python 3 :class:`bytes`. - When used in Unicode context, |ASN.1| type assumes "|encoding|" serialisation. + |ASN.1| class is based on :class:`~pyasn1.type.base.SimpleAsn1Type`, its + objects are immutable and duck-type Python 2 :class:`str` or + Python 3 :class:`bytes`. When used in Unicode context, |ASN.1| type + assumes "|encoding|" serialisation. Keyword Args ------------ - value: :class:`str`, :class:`bytes` or |ASN.1| object - string (Python 2) or bytes (Python 3), alternatively unicode object - (Python 2) or string (Python 3) representing character string to be - serialised into octets (note `encoding` parameter) or |ASN.1| object. + value: :class:`unicode`, :class:`str`, :class:`bytes` or |ASN.1| object + class:`str` (Python 2) or :class:`bytes` (Python 3), alternatively + class:`unicode` object (Python 2) or :class:`str` (Python 3) + representing character string to be serialised into octets + (note `encoding` parameter) or |ASN.1| object. + If `value` is not given, schema object will be created. tagSet: :py:class:`~pyasn1.type.tag.TagSet` Object representing non-default ASN.1 tag(s) subtypeSpec: :py:class:`~pyasn1.type.constraint.ConstraintsIntersection` - Object representing non-default ASN.1 subtype constraint(s) + Object representing non-default ASN.1 subtype constraint(s). Constraints + verification for |ASN.1| type occurs automatically on object + instantiation. encoding: :py:class:`str` Unicode codec ID to encode/decode :class:`unicode` (Python 2) or @@ -754,7 +772,7 @@ class OctetString(base.AbstractSimpleAsn1Item): Raises ------ - :py:class:`~pyasn1.error.PyAsn1Error` + ~pyasn1.error.ValueConstraintError, ~pyasn1.error.PyAsn1Error On constraint violation or bad initializer. Examples @@ -786,7 +804,7 @@ class OctetString(base.AbstractSimpleAsn1Item): subtypeSpec = constraint.ConstraintsIntersection() # Optimization for faster codec lookup - typeId = base.AbstractSimpleAsn1Item.getTypeId() + typeId = base.SimpleAsn1Type.getTypeId() defaultBinValue = defaultHexValue = noValue encoding = 'iso-8859-1' @@ -816,26 +834,33 @@ class OctetString(base.AbstractSimpleAsn1Item): if 'encoding' not in kwargs: kwargs['encoding'] = self.encoding - base.AbstractSimpleAsn1Item.__init__(self, value, **kwargs) + base.SimpleAsn1Type.__init__(self, value, **kwargs) if sys.version_info[0] <= 2: def prettyIn(self, value): if isinstance(value, str): return value + elif isinstance(value, unicode): try: return value.encode(self.encoding) + except (LookupError, UnicodeEncodeError): - raise error.PyAsn1Error( - "Can't encode string '%s' with codec %s" % (value, self.encoding) + exc = sys.exc_info()[1] + raise error.PyAsn1UnicodeEncodeError( + "Can't encode string '%s' with codec " + "%s" % (value, self.encoding), exc ) + elif isinstance(value, (tuple, list)): try: return ''.join([chr(x) for x in value]) + except ValueError: raise error.PyAsn1Error( "Bad %s initializer '%s'" % (self.__class__.__name__, value) ) + else: return str(value) @@ -847,8 +872,10 @@ class OctetString(base.AbstractSimpleAsn1Item): return self._value.decode(self.encoding) except UnicodeDecodeError: - raise error.PyAsn1Error( - "Can't decode string '%s' with codec %s" % (self._value, self.encoding) + exc = sys.exc_info()[1] + raise error.PyAsn1UnicodeDecodeError( + "Can't decode string '%s' with codec " + "%s" % (self._value, self.encoding), exc ) def asOctets(self): @@ -861,19 +888,26 @@ class OctetString(base.AbstractSimpleAsn1Item): def prettyIn(self, value): if isinstance(value, bytes): return value + elif isinstance(value, str): try: return value.encode(self.encoding) + except UnicodeEncodeError: - raise error.PyAsn1Error( - "Can't encode string '%s' with '%s' codec" % (value, self.encoding) + exc = sys.exc_info()[1] + raise error.PyAsn1UnicodeEncodeError( + "Can't encode string '%s' with '%s' " + "codec" % (value, self.encoding), exc ) elif isinstance(value, OctetString): # a shortcut, bytes() would work the same way return value.asOctets() - elif isinstance(value, base.AbstractSimpleAsn1Item): # this mostly targets Integer objects + + elif isinstance(value, base.SimpleAsn1Type): # this mostly targets Integer objects return self.prettyIn(str(value)) + elif isinstance(value, (tuple, list)): return self.prettyIn(bytes(value)) + else: return bytes(value) @@ -882,8 +916,11 @@ class OctetString(base.AbstractSimpleAsn1Item): return self._value.decode(self.encoding) except UnicodeDecodeError: - raise error.PyAsn1Error( - "Can't decode string '%s' with '%s' codec at '%s'" % (self._value, self.encoding, self.__class__.__name__) + exc = sys.exc_info()[1] + raise error.PyAsn1UnicodeDecodeError( + "Can't decode string '%s' with '%s' codec at " + "'%s'" % (self._value, self.encoding, + self.__class__.__name__), exc ) def __bytes__(self): @@ -1028,19 +1065,22 @@ class OctetString(base.AbstractSimpleAsn1Item): class Null(OctetString): """Create |ASN.1| schema or value object. - |ASN.1| objects are immutable and duck-type Python :class:`str` objects (always empty). + |ASN.1| class is based on :class:`~pyasn1.type.base.SimpleAsn1Type`, its + objects are immutable and duck-type Python :class:`str` objects + (always empty). Keyword Args ------------ - value: :class:`str` or :py:class:`~pyasn1.type.univ.Null` object - Python empty string literal or any object that evaluates to `False` + value: :class:`str` or |ASN.1| object + Python empty :class:`str` literal or any object that evaluates to :obj:`False` + If `value` is not given, schema object will be created. tagSet: :py:class:`~pyasn1.type.tag.TagSet` Object representing non-default ASN.1 tag(s) Raises ------ - :py:class:`~pyasn1.error.PyAsn1Error` + ~pyasn1.error.ValueConstraintError, ~pyasn1.error.PyAsn1Error On constraint violation or bad initializer. Examples @@ -1081,25 +1121,30 @@ else: numericTypes = intTypes + (float,) -class ObjectIdentifier(base.AbstractSimpleAsn1Item): +class ObjectIdentifier(base.SimpleAsn1Type): """Create |ASN.1| schema or value object. - |ASN.1| objects are immutable and duck-type Python :class:`tuple` objects (tuple of non-negative integers). + |ASN.1| class is based on :class:`~pyasn1.type.base.SimpleAsn1Type`, its + objects are immutable and duck-type Python :class:`tuple` objects + (tuple of non-negative integers). Keyword Args ------------ value: :class:`tuple`, :class:`str` or |ASN.1| object - Python sequence of :class:`int` or string literal or |ASN.1| object. + Python sequence of :class:`int` or :class:`str` literal or |ASN.1| object. + If `value` is not given, schema object will be created. tagSet: :py:class:`~pyasn1.type.tag.TagSet` Object representing non-default ASN.1 tag(s) subtypeSpec: :py:class:`~pyasn1.type.constraint.ConstraintsIntersection` - Object representing non-default ASN.1 subtype constraint(s) + Object representing non-default ASN.1 subtype constraint(s). Constraints + verification for |ASN.1| type occurs automatically on object + instantiation. Raises ------ - :py:class:`~pyasn1.error.PyAsn1Error` + ~pyasn1.error.ValueConstraintError, ~pyasn1.error.PyAsn1Error On constraint violation or bad initializer. Examples @@ -1131,7 +1176,7 @@ class ObjectIdentifier(base.AbstractSimpleAsn1Item): subtypeSpec = constraint.ConstraintsIntersection() # Optimization for faster codec lookup - typeId = base.AbstractSimpleAsn1Item.getTypeId() + typeId = base.SimpleAsn1Type.getTypeId() def __add__(self, other): return self.clone(self._value + other) @@ -1173,8 +1218,8 @@ class ObjectIdentifier(base.AbstractSimpleAsn1Item): Returns ------- : :class:`bool` - :class:`True` if this |ASN.1| object is a parent (e.g. prefix) of the other |ASN.1| object - or :class:`False` otherwise. + :obj:`True` if this |ASN.1| object is a parent (e.g. prefix) of the other |ASN.1| object + or :obj:`False` otherwise. """ l = len(self) if l <= len(other): @@ -1214,10 +1259,11 @@ class ObjectIdentifier(base.AbstractSimpleAsn1Item): return '.'.join([str(x) for x in value]) -class Real(base.AbstractSimpleAsn1Item): +class Real(base.SimpleAsn1Type): """Create |ASN.1| schema or value object. - |ASN.1| objects are immutable and duck-type Python :class:`float` objects. + |ASN.1| class is based on :class:`~pyasn1.type.base.SimpleAsn1Type`, its + objects are immutable and duck-type Python :class:`float` objects. Additionally, |ASN.1| objects behave like a :class:`tuple` in which case its elements are mantissa, base and exponent. @@ -1225,17 +1271,20 @@ class Real(base.AbstractSimpleAsn1Item): ------------ value: :class:`tuple`, :class:`float` or |ASN.1| object Python sequence of :class:`int` (representing mantissa, base and - exponent) or float instance or *Real* class instance. + exponent) or :class:`float` instance or |ASN.1| object. + If `value` is not given, schema object will be created. tagSet: :py:class:`~pyasn1.type.tag.TagSet` Object representing non-default ASN.1 tag(s) subtypeSpec: :py:class:`~pyasn1.type.constraint.ConstraintsIntersection` - Object representing non-default ASN.1 subtype constraint(s) + Object representing non-default ASN.1 subtype constraint(s). Constraints + verification for |ASN.1| type occurs automatically on object + instantiation. Raises ------ - :py:class:`~pyasn1.error.PyAsn1Error` + ~pyasn1.error.ValueConstraintError, ~pyasn1.error.PyAsn1Error On constraint violation or bad initializer. Examples @@ -1278,7 +1327,7 @@ class Real(base.AbstractSimpleAsn1Item): subtypeSpec = constraint.ConstraintsIntersection() # Optimization for faster codec lookup - typeId = base.AbstractSimpleAsn1Item.getTypeId() + typeId = base.SimpleAsn1Type.getTypeId() @staticmethod def __normalizeBase10(value): @@ -1342,8 +1391,8 @@ class Real(base.AbstractSimpleAsn1Item): Returns ------- : :class:`bool` - :class:`True` if calling object represents plus infinity - or :class:`False` otherwise. + :obj:`True` if calling object represents plus infinity + or :obj:`False` otherwise. """ return self._value == self._plusInf @@ -1355,8 +1404,8 @@ class Real(base.AbstractSimpleAsn1Item): Returns ------- : :class:`bool` - :class:`True` if calling object represents minus infinity - or :class:`False` otherwise. + :obj:`True` if calling object represents minus infinity + or :obj:`False` otherwise. """ return self._value == self._minusInf @@ -1479,7 +1528,7 @@ class Real(base.AbstractSimpleAsn1Item): def __bool__(self): return bool(float(self)) - __hash__ = base.AbstractSimpleAsn1Item.__hash__ + __hash__ = base.SimpleAsn1Type.__hash__ def __getitem__(self, idx): if self._value in self._inf: @@ -1500,27 +1549,31 @@ class Real(base.AbstractSimpleAsn1Item): class Enumerated(Integer): - """Create |ASN.1| type or object. + """Create |ASN.1| schema or value object. - |ASN.1| objects are immutable and duck-type Python :class:`int` objects. + |ASN.1| class is based on :class:`~pyasn1.type.base.SimpleAsn1Type`, its + objects are immutable and duck-type Python :class:`int` objects. Keyword Args ------------ value: :class:`int`, :class:`str` or |ASN.1| object - Python integer or string literal or |ASN.1| class instance. + Python :class:`int` or :class:`str` literal or |ASN.1| object. + If `value` is not given, schema object will be created. tagSet: :py:class:`~pyasn1.type.tag.TagSet` Object representing non-default ASN.1 tag(s) subtypeSpec: :py:class:`~pyasn1.type.constraint.ConstraintsIntersection` - Object representing non-default ASN.1 subtype constraint(s) + Object representing non-default ASN.1 subtype constraint(s). Constraints + verification for |ASN.1| type occurs automatically on object + instantiation. namedValues: :py:class:`~pyasn1.type.namedval.NamedValues` Object representing non-default symbolic aliases for numbers Raises ------ - :py:class:`~pyasn1.error.PyAsn1Error` + ~pyasn1.error.ValueConstraintError, ~pyasn1.error.PyAsn1Error On constraint violation or bad initializer. Examples @@ -1566,10 +1619,11 @@ class Enumerated(Integer): # "Structured" ASN.1 types -class SequenceOfAndSetOfBase(base.AbstractConstructedAsn1Item): - """Create |ASN.1| type. +class SequenceOfAndSetOfBase(base.ConstructedAsn1Type): + """Create |ASN.1| schema or value object. - |ASN.1| objects are mutable and duck-type Python :class:`list` objects. + |ASN.1| class is based on :class:`~pyasn1.type.base.ConstructedAsn1Type`, + its objects are mutable and duck-type Python :class:`list` objects. Keyword Args ------------ @@ -1580,10 +1634,9 @@ class SequenceOfAndSetOfBase(base.AbstractConstructedAsn1Item): Object representing non-default ASN.1 tag(s) subtypeSpec: :py:class:`~pyasn1.type.constraint.ConstraintsIntersection` - Object representing non-default ASN.1 subtype constraint(s) - - sizeSpec: :py:class:`~pyasn1.type.constraint.ConstraintsIntersection` - Object representing collection size constraint + Object representing non-default ASN.1 subtype constraint(s). Constraints + verification for |ASN.1| type can only occur on explicit + `.isInconsistent` call. Examples -------- @@ -1605,12 +1658,14 @@ class SequenceOfAndSetOfBase(base.AbstractConstructedAsn1Item): # support positional params for backward compatibility if args: for key, value in zip(('componentType', 'tagSet', - 'subtypeSpec', 'sizeSpec'), args): + 'subtypeSpec'), args): if key in kwargs: raise error.PyAsn1Error('Conflicting positional and keyword params!') kwargs['componentType'] = value - base.AbstractConstructedAsn1Item.__init__(self, **kwargs) + self._componentValues = noValue + + base.ConstructedAsn1Type.__init__(self, **kwargs) # Python list protocol @@ -1628,24 +1683,36 @@ class SequenceOfAndSetOfBase(base.AbstractConstructedAsn1Item): except error.PyAsn1Error: raise IndexError(sys.exc_info()[1]) - def clear(self): - self._componentValues = [] - def append(self, value): - self[len(self)] = value + if self._componentValues is noValue: + pos = 0 + + else: + pos = len(self._componentValues) + + self[pos] = value def count(self, value): - return self._componentValues.count(value) + return list(self._componentValues.values()).count(value) def extend(self, values): for value in values: self.append(value) + if self._componentValues is noValue: + self._componentValues = {} + def index(self, value, start=0, stop=None): if stop is None: stop = len(self) + + indices, values = zip(*self._componentValues.items()) + + # TODO: remove when Py2.5 support is gone + values = list(values) + try: - return self._componentValues.index(value, start, stop) + return indices[values.index(value, start, stop)] except error.PyAsn1Error: raise ValueError(sys.exc_info()[1]) @@ -1654,15 +1721,24 @@ class SequenceOfAndSetOfBase(base.AbstractConstructedAsn1Item): self._componentValues.reverse() def sort(self, key=None, reverse=False): - self._componentValues.sort(key=key, reverse=reverse) + self._componentValues = dict( + enumerate(sorted(self._componentValues.values(), + key=key, reverse=reverse))) + + def __len__(self): + if self._componentValues is noValue or not self._componentValues: + return 0 + + return max(self._componentValues) + 1 def __iter__(self): - return iter(self._componentValues) + for idx in range(0, len(self)): + yield self.getComponentByPosition(idx) def _cloneComponentValues(self, myClone, cloneValueFlag): - for idx, componentValue in enumerate(self._componentValues): + for idx, componentValue in self._componentValues.items(): if componentValue is not noValue: - if isinstance(componentValue, base.AbstractConstructedAsn1Item): + if isinstance(componentValue, base.ConstructedAsn1Type): myClone.setComponentByPosition( idx, componentValue.clone(cloneValueFlag=cloneValueFlag) ) @@ -1689,8 +1765,8 @@ class SequenceOfAndSetOfBase(base.AbstractConstructedAsn1Item): object instead of the requested component. instantiate: :class:`bool` - If `True` (default), inner component will be automatically instantiated. - If 'False' either existing component or the `noValue` object will be + If :obj:`True` (default), inner component will be automatically instantiated. + If :obj:`False` either existing component or the :class:`NoValue` object will be returned. Returns @@ -1735,10 +1811,21 @@ class SequenceOfAndSetOfBase(base.AbstractConstructedAsn1Item): # returns noValue s.getComponentByPosition(0, instantiate=False) """ + if isinstance(idx, slice): + indices = tuple(range(len(self))) + return [self.getComponentByPosition(subidx, default, instantiate) + for subidx in indices[idx]] + + if idx < 0: + idx = len(self) + idx + if idx < 0: + raise error.PyAsn1Error( + 'SequenceOf/SetOf index is out of range') + try: componentValue = self._componentValues[idx] - except IndexError: + except (KeyError, error.PyAsn1Error): if not instantiate: return default @@ -1773,15 +1860,16 @@ class SequenceOfAndSetOfBase(base.AbstractConstructedAsn1Item): value: :class:`object` or :py:class:`~pyasn1.type.base.PyAsn1Item` derivative A Python value to initialize |ASN.1| component with (if *componentType* is set) or ASN.1 value object to assign to |ASN.1| component. + If `value` is not given, schema object will be set as a component. verifyConstraints: :class:`bool` - If `False`, skip constraints validation + If :obj:`False`, skip constraints validation matchTags: :class:`bool` - If `False`, skip component tags matching + If :obj:`False`, skip component tags matching matchConstraints: :class:`bool` - If `False`, skip component constraints matching + If :obj:`False`, skip component constraints matching Returns ------- @@ -1789,51 +1877,75 @@ class SequenceOfAndSetOfBase(base.AbstractConstructedAsn1Item): Raises ------ - IndexError: + ~pyasn1.error.ValueConstraintError, ~pyasn1.error.PyAsn1Error + On constraint violation or bad initializer + IndexError When idx > len(self) """ + if isinstance(idx, slice): + indices = tuple(range(len(self))) + startIdx = indices and indices[idx][0] or 0 + for subIdx, subValue in enumerate(value): + self.setComponentByPosition( + startIdx + subIdx, subValue, verifyConstraints, + matchTags, matchConstraints) + return self + + if idx < 0: + idx = len(self) + idx + if idx < 0: + raise error.PyAsn1Error( + 'SequenceOf/SetOf index is out of range') + componentType = self.componentType - try: - currentValue = self._componentValues[idx] - except IndexError: - currentValue = noValue + if self._componentValues is noValue: + componentValues = {} - if len(self._componentValues) < idx: - raise error.PyAsn1Error('Component index out of range') + else: + componentValues = self._componentValues + + currentValue = componentValues.get(idx, noValue) if value is noValue: if componentType is not None: value = componentType.clone() + elif currentValue is noValue: raise error.PyAsn1Error('Component type not defined') + elif not isinstance(value, base.Asn1Item): - if componentType is not None and isinstance(componentType, base.AbstractSimpleAsn1Item): + if (componentType is not None and + isinstance(componentType, base.SimpleAsn1Type)): value = componentType.clone(value=value) - elif currentValue is not noValue and isinstance(currentValue, base.AbstractSimpleAsn1Item): + + elif (currentValue is not noValue and + isinstance(currentValue, base.SimpleAsn1Type)): value = currentValue.clone(value=value) + else: - raise error.PyAsn1Error('Non-ASN.1 value %r and undefined component type at %r' % (value, self)) - elif componentType is not None: - if self.strictConstraints: - if not componentType.isSameTypeWith(value, matchTags, matchConstraints): - raise error.PyAsn1Error('Component value is tag-incompatible: %r vs %r' % (value, componentType)) - else: - if not componentType.isSuperTypeOf(value, matchTags, matchConstraints): - raise error.PyAsn1Error('Component value is tag-incompatible: %r vs %r' % (value, componentType)) + raise error.PyAsn1Error( + 'Non-ASN.1 value %r and undefined component' + ' type at %r' % (value, self)) - if verifyConstraints and value.isValue: - try: - self.subtypeSpec(value, idx) + elif componentType is not None and (matchTags or matchConstraints): + subtypeChecker = ( + self.strictConstraints and + componentType.isSameTypeWith or + componentType.isSuperTypeOf) - except error.PyAsn1Error: - exType, exValue, exTb = sys.exc_info() - raise exType('%s at %s' % (exValue, self.__class__.__name__)) + if not subtypeChecker(value, verifyConstraints and matchTags, + verifyConstraints and matchConstraints): + # TODO: we should wrap componentType with UnnamedType to carry + # additional properties associated with componentType + if componentType.typeId != Any.typeId: + raise error.PyAsn1Error( + 'Component value is tag-incompatible: %r vs ' + '%r' % (value, componentType)) - if currentValue is noValue: - self._componentValues.append(value) - else: - self._componentValues[idx] = value + componentValues[idx] = value + + self._componentValues = componentValues return self @@ -1842,16 +1954,44 @@ class SequenceOfAndSetOfBase(base.AbstractConstructedAsn1Item): if self.componentType is not None: return self.componentType.tagMap + @property + def components(self): + return [self._componentValues[idx] + for idx in sorted(self._componentValues)] + + def clear(self): + """Remove all components and become an empty |ASN.1| value object. + + Has the same effect on |ASN.1| object as it does on :class:`list` + built-in. + """ + self._componentValues = {} + return self + + def reset(self): + """Remove all components and become a |ASN.1| schema object. + + See :meth:`isValue` property for more information on the + distinction between value and schema objects. + """ + self._componentValues = noValue + return self + def prettyPrint(self, scope=0): scope += 1 representation = self.__class__.__name__ + ':\n' - for idx, componentValue in enumerate(self._componentValues): + + if not self.isValue: + return representation + + for idx, componentValue in enumerate(self): representation += ' ' * scope if (componentValue is noValue and self.componentType is not None): representation += '' else: representation += componentValue.prettyPrint(scope) + return representation def prettyPrintType(self, scope=0): @@ -1867,17 +2007,17 @@ class SequenceOfAndSetOfBase(base.AbstractConstructedAsn1Item): def isValue(self): """Indicate that |ASN.1| object represents ASN.1 value. - If *isValue* is `False` then this object represents just ASN.1 schema. + If *isValue* is :obj:`False` then this object represents just ASN.1 schema. - If *isValue* is `True` then, in addition to its ASN.1 schema features, - this object can also be used like a Python built-in object (e.g. `int`, - `str`, `dict` etc.). + If *isValue* is :obj:`True` then, in addition to its ASN.1 schema features, + this object can also be used like a Python built-in object + (e.g. :class:`int`, :class:`str`, :class:`dict` etc.). Returns ------- : :class:`bool` - :class:`False` if object represents just ASN.1 schema. - :class:`True` if object represents ASN.1 schema and can be used as a normal value. + :obj:`False` if object represents just ASN.1 schema. + :obj:`True` if object represents ASN.1 schema and can be used as a normal value. Note ---- @@ -1890,12 +2030,53 @@ class SequenceOfAndSetOfBase(base.AbstractConstructedAsn1Item): The PyASN1 value objects can **additionally** participate in many operations involving regular Python objects (e.g. arithmetic, comprehension etc). """ - for componentValue in self._componentValues: + if self._componentValues is noValue: + return False + + if len(self._componentValues) != len(self): + return False + + for componentValue in self._componentValues.values(): if componentValue is noValue or not componentValue.isValue: return False return True + @property + def isInconsistent(self): + """Run necessary checks to ensure |ASN.1| object consistency. + + Default action is to verify |ASN.1| object against constraints imposed + by `subtypeSpec`. + + Raises + ------ + :py:class:`~pyasn1.error.PyAsn1tError` on any inconsistencies found + """ + if self.componentType is noValue or not self.subtypeSpec: + return False + + if self._componentValues is noValue: + return True + + mapping = {} + + for idx, value in self._componentValues.items(): + # Absent fields are not in the mapping + if value is noValue: + continue + + mapping[idx] = value + + try: + # Represent SequenceOf/SetOf as a bare dict to constraints chain + self.subtypeSpec(mapping) + + except error.PyAsn1Error: + exc = sys.exc_info()[1] + return exc + + return False class SequenceOf(SequenceOfAndSetOfBase): __doc__ = SequenceOfAndSetOfBase.__doc__ @@ -1916,10 +2097,6 @@ class SequenceOf(SequenceOfAndSetOfBase): #: imposing constraints on |ASN.1| type initialization values. subtypeSpec = constraint.ConstraintsIntersection() - #: Default :py:class:`~pyasn1.type.constraint.ConstraintsIntersection` - #: object imposing size constraint on |ASN.1| objects - sizeSpec = constraint.ConstraintsIntersection() - # Disambiguation ASN.1 types identification typeId = SequenceOfAndSetOfBase.getTypeId() @@ -1943,18 +2120,15 @@ class SetOf(SequenceOfAndSetOfBase): #: imposing constraints on |ASN.1| type initialization values. subtypeSpec = constraint.ConstraintsIntersection() - #: Default :py:class:`~pyasn1.type.constraint.ConstraintsIntersection` - #: object imposing size constraint on |ASN.1| objects - sizeSpec = constraint.ConstraintsIntersection() - # Disambiguation ASN.1 types identification typeId = SequenceOfAndSetOfBase.getTypeId() -class SequenceAndSetBase(base.AbstractConstructedAsn1Item): - """Create |ASN.1| type. +class SequenceAndSetBase(base.ConstructedAsn1Type): + """Create |ASN.1| schema or value object. - |ASN.1| objects are mutable and duck-type Python :class:`dict` objects. + |ASN.1| class is based on :class:`~pyasn1.type.base.ConstructedAsn1Type`, + its objects are mutable and duck-type Python :class:`dict` objects. Keyword Args ------------ @@ -1965,10 +2139,9 @@ class SequenceAndSetBase(base.AbstractConstructedAsn1Item): Object representing non-default ASN.1 tag(s) subtypeSpec: :py:class:`~pyasn1.type.constraint.ConstraintsIntersection` - Object representing non-default ASN.1 subtype constraint(s) - - sizeSpec: :py:class:`~pyasn1.type.constraint.ConstraintsIntersection` - Object representing collection size constraint + Object representing non-default ASN.1 subtype constraint(s). Constraints + verification for |ASN.1| type can only occur on explicit + `.isInconsistent` call. Examples -------- @@ -2042,8 +2215,12 @@ class SequenceAndSetBase(base.AbstractConstructedAsn1Item): def __init__(self, **kwargs): - base.AbstractConstructedAsn1Item.__init__(self, **kwargs) + base.ConstructedAsn1Type.__init__(self, **kwargs) self._componentTypeLen = len(self.componentType) + if self._componentTypeLen: + self._componentValues = [] + else: + self._componentValues = noValue self._dynamicNames = self._componentTypeLen or self.DynamicNames() def __getitem__(self, idx): @@ -2086,6 +2263,9 @@ class SequenceAndSetBase(base.AbstractConstructedAsn1Item): else: return key in self._dynamicNames + def __len__(self): + return len(self._componentValues) + def __iter__(self): return iter(self.componentType or self._dynamicNames) @@ -2112,13 +2292,36 @@ class SequenceAndSetBase(base.AbstractConstructedAsn1Item): self[k] = mappingValue[k] def clear(self): + """Remove all components and become an empty |ASN.1| value object. + + Has the same effect on |ASN.1| object as it does on :class:`dict` + built-in. + """ self._componentValues = [] self._dynamicNames = self.DynamicNames() + return self + + def reset(self): + """Remove all components and become a |ASN.1| schema object. + + See :meth:`isValue` property for more information on the + distinction between value and schema objects. + """ + self._componentValues = noValue + self._dynamicNames = self.DynamicNames() + return self + + @property + def components(self): + return self._componentValues def _cloneComponentValues(self, myClone, cloneValueFlag): + if self._componentValues is noValue: + return + for idx, componentValue in enumerate(self._componentValues): if componentValue is not noValue: - if isinstance(componentValue, base.AbstractConstructedAsn1Item): + if isinstance(componentValue, base.ConstructedAsn1Type): myClone.setComponentByPosition( idx, componentValue.clone(cloneValueFlag=cloneValueFlag) ) @@ -2142,14 +2345,16 @@ class SequenceAndSetBase(base.AbstractConstructedAsn1Item): object instead of the requested component. instantiate: :class:`bool` - If `True` (default), inner component will be automatically instantiated. - If 'False' either existing component or the `noValue` object will be - returned. + If :obj:`True` (default), inner component will be automatically + instantiated. + If :obj:`False` either existing component or the :class:`NoValue` + object will be returned. Returns ------- : :py:class:`~pyasn1.type.base.PyAsn1Item` - Instantiate |ASN.1| component type or return existing component value + Instantiate |ASN.1| component type or return existing + component value """ if self._componentTypeLen: idx = self.componentType.getPositionByName(name) @@ -2180,15 +2385,16 @@ class SequenceAndSetBase(base.AbstractConstructedAsn1Item): value: :class:`object` or :py:class:`~pyasn1.type.base.PyAsn1Item` derivative A Python value to initialize |ASN.1| component with (if *componentType* is set) or ASN.1 value object to assign to |ASN.1| component. + If `value` is not given, schema object will be set as a component. verifyConstraints: :class:`bool` - If `False`, skip constraints validation + If :obj:`False`, skip constraints validation matchTags: :class:`bool` - If `False`, skip component tags matching + If :obj:`False`, skip component tags matching matchConstraints: :class:`bool` - If `False`, skip component constraints matching + If :obj:`False`, skip component constraints matching Returns ------- @@ -2226,9 +2432,10 @@ class SequenceAndSetBase(base.AbstractConstructedAsn1Item): object instead of the requested component. instantiate: :class:`bool` - If `True` (default), inner component will be automatically instantiated. - If 'False' either existing component or the `noValue` object will be - returned. + If :obj:`True` (default), inner component will be automatically + instantiated. + If :obj:`False` either existing component or the :class:`NoValue` + object will be returned. Returns ------- @@ -2275,7 +2482,11 @@ class SequenceAndSetBase(base.AbstractConstructedAsn1Item): s.getComponentByPosition(0, instantiate=False) """ try: - componentValue = self._componentValues[idx] + if self._componentValues is noValue: + componentValue = noValue + + else: + componentValue = self._componentValues[idx] except IndexError: componentValue = noValue @@ -2317,15 +2528,16 @@ class SequenceAndSetBase(base.AbstractConstructedAsn1Item): value: :class:`object` or :py:class:`~pyasn1.type.base.PyAsn1Item` derivative A Python value to initialize |ASN.1| component with (if *componentType* is set) or ASN.1 value object to assign to |ASN.1| component. + If `value` is not given, schema object will be set as a component. verifyConstraints : :class:`bool` - If `False`, skip constraints validation + If :obj:`False`, skip constraints validation matchTags: :class:`bool` - If `False`, skip component tags matching + If :obj:`False`, skip component tags matching matchConstraints: :class:`bool` - If `False`, skip component constraints matching + If :obj:`False`, skip component constraints matching Returns ------- @@ -2334,8 +2546,14 @@ class SequenceAndSetBase(base.AbstractConstructedAsn1Item): componentType = self.componentType componentTypeLen = self._componentTypeLen + if self._componentValues is noValue: + componentValues = [] + + else: + componentValues = self._componentValues + try: - currentValue = self._componentValues[idx] + currentValue = componentValues[idx] except IndexError: currentValue = noValue @@ -2343,11 +2561,13 @@ class SequenceAndSetBase(base.AbstractConstructedAsn1Item): if componentTypeLen < idx: raise error.PyAsn1Error('component index out of range') - self._componentValues = [noValue] * componentTypeLen + componentValues = [noValue] * componentTypeLen if value is noValue: if componentTypeLen: - value = componentType.getTypeByPosition(idx).clone() + value = componentType.getTypeByPosition(idx) + if isinstance(value, base.ConstructedAsn1Type): + value = value.clone(cloneValueFlag=componentType[idx].isDefaulted) elif currentValue is noValue: raise error.PyAsn1Error('Component type not defined') @@ -2355,64 +2575,61 @@ class SequenceAndSetBase(base.AbstractConstructedAsn1Item): elif not isinstance(value, base.Asn1Item): if componentTypeLen: subComponentType = componentType.getTypeByPosition(idx) - if isinstance(subComponentType, base.AbstractSimpleAsn1Item): + if isinstance(subComponentType, base.SimpleAsn1Type): value = subComponentType.clone(value=value) else: raise error.PyAsn1Error('%s can cast only scalar values' % componentType.__class__.__name__) - elif currentValue is not noValue and isinstance(currentValue, base.AbstractSimpleAsn1Item): + elif currentValue is not noValue and isinstance(currentValue, base.SimpleAsn1Type): value = currentValue.clone(value=value) else: raise error.PyAsn1Error('%s undefined component type' % componentType.__class__.__name__) - elif (matchTags or matchConstraints) and componentTypeLen: + elif ((verifyConstraints or matchTags or matchConstraints) and + componentTypeLen): subComponentType = componentType.getTypeByPosition(idx) if subComponentType is not noValue: subtypeChecker = (self.strictConstraints and subComponentType.isSameTypeWith or subComponentType.isSuperTypeOf) - if not subtypeChecker(value, matchTags, matchConstraints): + if not subtypeChecker(value, verifyConstraints and matchTags, + verifyConstraints and matchConstraints): if not componentType[idx].openType: raise error.PyAsn1Error('Component value is tag-incompatible: %r vs %r' % (value, componentType)) - if verifyConstraints and value.isValue: - try: - self.subtypeSpec(value, idx) - - except error.PyAsn1Error: - exType, exValue, exTb = sys.exc_info() - raise exType('%s at %s' % (exValue, self.__class__.__name__)) - if componentTypeLen or idx in self._dynamicNames: - self._componentValues[idx] = value + componentValues[idx] = value - elif len(self._componentValues) == idx: - self._componentValues.append(value) + elif len(componentValues) == idx: + componentValues.append(value) self._dynamicNames.addField(idx) else: raise error.PyAsn1Error('Component index out of range') + self._componentValues = componentValues + return self @property def isValue(self): """Indicate that |ASN.1| object represents ASN.1 value. - If *isValue* is `False` then this object represents just ASN.1 schema. + If *isValue* is :obj:`False` then this object represents just ASN.1 schema. - If *isValue* is `True` then, in addition to its ASN.1 schema features, - this object can also be used like a Python built-in object (e.g. `int`, - `str`, `dict` etc.). + If *isValue* is :obj:`True` then, in addition to its ASN.1 schema features, + this object can also be used like a Python built-in object (e.g. + :class:`int`, :class:`str`, :class:`dict` etc.). Returns ------- : :class:`bool` - :class:`False` if object represents just ASN.1 schema. - :class:`True` if object represents ASN.1 schema and can be used as a normal value. + :obj:`False` if object represents just ASN.1 schema. + :obj:`True` if object represents ASN.1 schema and can be used as a + normal value. Note ---- @@ -2424,7 +2641,16 @@ class SequenceAndSetBase(base.AbstractConstructedAsn1Item): The PyASN1 value objects can **additionally** participate in many operations involving regular Python objects (e.g. arithmetic, comprehension etc). + + It is sufficient for |ASN.1| objects to have all non-optional and non-defaulted + components being value objects to be considered as a value objects as a whole. + In other words, even having one or more optional components not turned into + value objects, |ASN.1| object is still considered as a value object. Defaulted + components are normally value objects by default. """ + if self._componentValues is noValue: + return False + componentType = self.componentType if componentType: @@ -2446,6 +2672,44 @@ class SequenceAndSetBase(base.AbstractConstructedAsn1Item): return True + @property + def isInconsistent(self): + """Run necessary checks to ensure |ASN.1| object consistency. + + Default action is to verify |ASN.1| object against constraints imposed + by `subtypeSpec`. + + Raises + ------ + :py:class:`~pyasn1.error.PyAsn1tError` on any inconsistencies found + """ + if self.componentType is noValue or not self.subtypeSpec: + return False + + if self._componentValues is noValue: + return True + + mapping = {} + + for idx, value in enumerate(self._componentValues): + # Absent fields are not in the mapping + if value is noValue: + continue + + name = self.componentType.getNameByPosition(idx) + + mapping[name] = value + + try: + # Represent Sequence/Set as a bare dict to constraints chain + self.subtypeSpec(mapping) + + except error.PyAsn1Error: + exc = sys.exc_info()[1] + return exc + + return False + def prettyPrint(self, scope=0): """Return an object representation string. @@ -2495,7 +2759,6 @@ class SequenceAndSetBase(base.AbstractConstructedAsn1Item): if self._componentTypeLen: return self.componentType[idx].name - class Sequence(SequenceAndSetBase): __doc__ = SequenceAndSetBase.__doc__ @@ -2511,10 +2774,6 @@ class Sequence(SequenceAndSetBase): #: imposing constraints on |ASN.1| type initialization values. subtypeSpec = constraint.ConstraintsIntersection() - #: Default :py:class:`~pyasn1.type.constraint.ConstraintsIntersection` - #: object imposing constraints on |ASN.1| objects - sizeSpec = constraint.ConstraintsIntersection() - #: Default collection of ASN.1 types of component (e.g. :py:class:`~pyasn1.type.namedtype.NamedType`) #: object imposing size constraint on |ASN.1| objects componentType = namedtype.NamedTypes() @@ -2554,10 +2813,6 @@ class Set(SequenceAndSetBase): #: imposing constraints on |ASN.1| type initialization values. subtypeSpec = constraint.ConstraintsIntersection() - #: Default :py:class:`~pyasn1.type.constraint.ConstraintsIntersection` - #: object imposing constraints on |ASN.1| objects - sizeSpec = constraint.ConstraintsIntersection() - # Disambiguation ASN.1 types identification typeId = SequenceAndSetBase.getTypeId() @@ -2581,9 +2836,10 @@ class Set(SequenceAndSetBase): object instead of the requested component. instantiate: :class:`bool` - If `True` (default), inner component will be automatically instantiated. - If 'False' either existing component or the `noValue` object will be - returned. + If :obj:`True` (default), inner component will be automatically + instantiated. + If :obj:`False` either existing component or the :class:`noValue` + object will be returned. Returns ------- @@ -2619,18 +2875,19 @@ class Set(SequenceAndSetBase): value: :class:`object` or :py:class:`~pyasn1.type.base.PyAsn1Item` derivative A Python value to initialize |ASN.1| component with (if *componentType* is set) or ASN.1 value object to assign to |ASN.1| component. + If `value` is not given, schema object will be set as a component. verifyConstraints : :class:`bool` - If `False`, skip constraints validation + If :obj:`False`, skip constraints validation matchTags: :class:`bool` - If `False`, skip component tags matching + If :obj:`False`, skip component tags matching matchConstraints: :class:`bool` - If `False`, skip component constraints matching + If :obj:`False`, skip component constraints matching innerFlag: :class:`bool` - If `True`, search for matching *tagSet* recursively. + If :obj:`True`, search for matching *tagSet* recursively. Returns ------- @@ -2662,9 +2919,10 @@ class Set(SequenceAndSetBase): class Choice(Set): - """Create |ASN.1| type. + """Create |ASN.1| schema or value object. - |ASN.1| objects are mutable and duck-type Python :class:`dict` objects. + |ASN.1| class is based on :class:`~pyasn1.type.base.ConstructedAsn1Type`, + its objects are mutable and duck-type Python :class:`list` objects. Keyword Args ------------ @@ -2675,10 +2933,9 @@ class Choice(Set): Object representing non-default ASN.1 tag(s) subtypeSpec: :py:class:`~pyasn1.type.constraint.ConstraintsIntersection` - Object representing non-default ASN.1 subtype constraint(s) - - sizeSpec: :py:class:`~pyasn1.type.constraint.ConstraintsIntersection` - Object representing collection size constraint + Object representing non-default ASN.1 subtype constraint(s). Constraints + verification for |ASN.1| type can only occur on explicit + `.isInconsistent` call. Examples -------- @@ -2718,11 +2975,7 @@ class Choice(Set): #: Set (on class, not on instance) or return a #: :py:class:`~pyasn1.type.constraint.ConstraintsIntersection` object #: imposing constraints on |ASN.1| type initialization values. - subtypeSpec = constraint.ConstraintsIntersection() - - #: Default :py:class:`~pyasn1.type.constraint.ConstraintsIntersection` - #: object imposing size constraint on |ASN.1| objects - sizeSpec = constraint.ConstraintsIntersection( + subtypeSpec = constraint.ConstraintsIntersection( constraint.ValueSizeConstraint(1, 1) ) @@ -2795,7 +3048,7 @@ class Choice(Set): if self._currentIdx is not None: yield self.componentType[self._currentIdx].getName(), self[self._currentIdx] - def verifySizeSpec(self): + def checkConsistency(self): if self._currentIdx is None: raise error.PyAsn1Error('Component not chosen') @@ -2809,7 +3062,7 @@ class Choice(Set): tagSet = component.effectiveTagSet else: tagSet = component.tagSet - if isinstance(component, base.AbstractConstructedAsn1Item): + if isinstance(component, base.ConstructedAsn1Type): myClone.setComponentByType( tagSet, component.clone(cloneValueFlag=cloneValueFlag) ) @@ -2847,15 +3100,16 @@ class Choice(Set): A Python value to initialize |ASN.1| component with (if *componentType* is set) or ASN.1 value object to assign to |ASN.1| component. Once a new value is set to *idx* component, previous value is dropped. + If `value` is not given, schema object will be set as a component. verifyConstraints : :class:`bool` - If `False`, skip constraints validation + If :obj:`False`, skip constraints validation matchTags: :class:`bool` - If `False`, skip component tags matching + If :obj:`False`, skip component tags matching matchConstraints: :class:`bool` - If `False`, skip component constraints matching + If :obj:`False`, skip component constraints matching Returns ------- @@ -2925,17 +3179,18 @@ class Choice(Set): def isValue(self): """Indicate that |ASN.1| object represents ASN.1 value. - If *isValue* is `False` then this object represents just ASN.1 schema. + If *isValue* is :obj:`False` then this object represents just ASN.1 schema. - If *isValue* is `True` then, in addition to its ASN.1 schema features, - this object can also be used like a Python built-in object (e.g. `int`, - `str`, `dict` etc.). + If *isValue* is :obj:`True` then, in addition to its ASN.1 schema features, + this object can also be used like a Python built-in object (e.g. + :class:`int`, :class:`str`, :class:`dict` etc.). Returns ------- : :class:`bool` - :class:`False` if object represents just ASN.1 schema. - :class:`True` if object represents ASN.1 schema and can be used as a normal value. + :obj:`False` if object represents just ASN.1 schema. + :obj:`True` if object represents ASN.1 schema and can be used as a normal + value. Note ---- @@ -2957,7 +3212,7 @@ class Choice(Set): def clear(self): self._currentIdx = None - Set.clear(self) + return Set.clear(self) # compatibility stubs @@ -2968,22 +3223,27 @@ class Choice(Set): class Any(OctetString): """Create |ASN.1| schema or value object. - |ASN.1| objects are immutable and duck-type Python 2 :class:`str` or Python 3 - :class:`bytes`. When used in Unicode context, |ASN.1| type assumes "|encoding|" - serialisation. + |ASN.1| class is based on :class:`~pyasn1.type.base.SimpleAsn1Type`, + its objects are immutable and duck-type Python 2 :class:`str` or Python 3 + :class:`bytes`. When used in Unicode context, |ASN.1| type assumes + "|encoding|" serialisation. Keyword Args ------------ - value: :class:`str`, :class:`bytes` or |ASN.1| object - string (Python 2) or bytes (Python 3), alternatively unicode object - (Python 2) or string (Python 3) representing character string to be - serialised into octets (note `encoding` parameter) or |ASN.1| object. + value: :class:`unicode`, :class:`str`, :class:`bytes` or |ASN.1| object + :class:`str` (Python 2) or :class:`bytes` (Python 3), alternatively + :class:`unicode` object (Python 2) or :class:`str` (Python 3) + representing character string to be serialised into octets (note + `encoding` parameter) or |ASN.1| object. + If `value` is not given, schema object will be created. tagSet: :py:class:`~pyasn1.type.tag.TagSet` Object representing non-default ASN.1 tag(s) subtypeSpec: :py:class:`~pyasn1.type.constraint.ConstraintsIntersection` - Object representing non-default ASN.1 subtype constraint(s) + Object representing non-default ASN.1 subtype constraint(s). Constraints + verification for |ASN.1| type occurs automatically on object + instantiation. encoding: :py:class:`str` Unicode codec ID to encode/decode :class:`unicode` (Python 2) or @@ -3000,7 +3260,7 @@ class Any(OctetString): Raises ------ - :py:class:`~pyasn1.error.PyAsn1Error` + ~pyasn1.error.ValueConstraintError, ~pyasn1.error.PyAsn1Error On constraint violation or bad initializer. Examples diff --git a/server/www/packages/packages-linux/x64/pyasn1/type/useful.py b/server/www/packages/packages-linux/x64/pyasn1/type/useful.py index 146916d..7536b95 100644 --- a/server/www/packages/packages-linux/x64/pyasn1/type/useful.py +++ b/server/www/packages/packages-linux/x64/pyasn1/type/useful.py @@ -1,7 +1,7 @@ # # This file is part of pyasn1 software. # -# Copyright (c) 2005-2018, Ilya Etingof +# Copyright (c) 2005-2019, Ilya Etingof # License: http://snmplabs.com/pyasn1/license.html # import datetime diff --git a/server/www/packages/packages-linux/x64/pymysql/__init__.py b/server/www/packages/packages-linux/x64/pymysql/__init__.py index b79b4b8..0cb5006 100644 --- a/server/www/packages/packages-linux/x64/pymysql/__init__.py +++ b/server/www/packages/packages-linux/x64/pymysql/__init__.py @@ -35,7 +35,7 @@ from .times import ( DateFromTicks, TimeFromTicks, TimestampFromTicks) -VERSION = (0, 9, 2, None) +VERSION = (0, 9, 3, None) if VERSION[3] is not None: VERSION_STRING = "%d.%d.%d_%s" % VERSION else: diff --git a/server/www/packages/packages-linux/x64/pymysql/_auth.py b/server/www/packages/packages-linux/x64/pymysql/_auth.py index bbb742d..199f36c 100644 --- a/server/www/packages/packages-linux/x64/pymysql/_auth.py +++ b/server/www/packages/packages-linux/x64/pymysql/_auth.py @@ -4,14 +4,22 @@ Implements auth methods from ._compat import text_type, PY2 from .constants import CLIENT from .err import OperationalError +from .util import byte2int, int2byte -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import serialization, hashes -from cryptography.hazmat.primitives.asymmetric import padding + +try: + from cryptography.hazmat.backends import default_backend + from cryptography.hazmat.primitives import serialization, hashes + from cryptography.hazmat.primitives.asymmetric import padding + _have_cryptography = True +except ImportError: + _have_cryptography = False from functools import partial import hashlib +import io import struct +import warnings DEBUG = False @@ -69,6 +77,8 @@ class RandStruct_323(object): def scramble_old_password(password, message): """Scramble for old_password""" + warnings.warn("old password (for MySQL <4.1) is used. Upgrade your password with newer auth method.\n" + "old password support will be removed in future PyMySQL version") hash_pass = _hash_password_323(password) hash_message = _hash_password_323(message[:SCRAMBLE_LENGTH_323]) hash_pass_n = struct.unpack(">LL", hash_pass) @@ -128,6 +138,8 @@ def sha2_rsa_encrypt(password, salt, public_key): Used for sha256_password and caching_sha2_password. """ + if not _have_cryptography: + raise RuntimeError("cryptography is required for sha256_password or caching_sha2_password") message = _xor_password(password + b'\0', salt) rsa_key = serialization.load_pem_public_key(public_key, default_backend()) return rsa_key.encrypt( diff --git a/server/www/packages/packages-linux/x64/pymysql/charset.py b/server/www/packages/packages-linux/x64/pymysql/charset.py index 968376c..07d8063 100644 --- a/server/www/packages/packages-linux/x64/pymysql/charset.py +++ b/server/www/packages/packages-linux/x64/pymysql/charset.py @@ -18,7 +18,7 @@ class Charset(object): @property def encoding(self): name = self.name - if name == 'utf8mb4': + if name in ('utf8mb4', 'utf8mb3'): return 'utf8' return name @@ -30,18 +30,18 @@ class Charset(object): class Charsets: def __init__(self): self._by_id = {} + self._by_name = {} def add(self, c): self._by_id[c.id] = c + if c.is_default: + self._by_name[c.name] = c def by_id(self, id): return self._by_id[id] def by_name(self, name): - name = name.lower() - for c in self._by_id.values(): - if c.name == name and c.is_default: - return c + return self._by_name.get(name.lower()) _charsets = Charsets() """ @@ -89,7 +89,6 @@ _charsets.add(Charset(31, 'latin1', 'latin1_german2_ci', '')) _charsets.add(Charset(32, 'armscii8', 'armscii8_general_ci', 'Yes')) _charsets.add(Charset(33, 'utf8', 'utf8_general_ci', 'Yes')) _charsets.add(Charset(34, 'cp1250', 'cp1250_czech_cs', '')) -_charsets.add(Charset(35, 'ucs2', 'ucs2_general_ci', 'Yes')) _charsets.add(Charset(36, 'cp866', 'cp866_general_ci', 'Yes')) _charsets.add(Charset(37, 'keybcs2', 'keybcs2_general_ci', 'Yes')) _charsets.add(Charset(38, 'macce', 'macce_general_ci', 'Yes')) @@ -108,13 +107,9 @@ _charsets.add(Charset(50, 'cp1251', 'cp1251_bin', '')) _charsets.add(Charset(51, 'cp1251', 'cp1251_general_ci', 'Yes')) _charsets.add(Charset(52, 'cp1251', 'cp1251_general_cs', '')) _charsets.add(Charset(53, 'macroman', 'macroman_bin', '')) -_charsets.add(Charset(54, 'utf16', 'utf16_general_ci', 'Yes')) -_charsets.add(Charset(55, 'utf16', 'utf16_bin', '')) _charsets.add(Charset(57, 'cp1256', 'cp1256_general_ci', 'Yes')) _charsets.add(Charset(58, 'cp1257', 'cp1257_bin', '')) _charsets.add(Charset(59, 'cp1257', 'cp1257_general_ci', 'Yes')) -_charsets.add(Charset(60, 'utf32', 'utf32_general_ci', 'Yes')) -_charsets.add(Charset(61, 'utf32', 'utf32_bin', '')) _charsets.add(Charset(63, 'binary', 'binary', 'Yes')) _charsets.add(Charset(64, 'armscii8', 'armscii8_bin', '')) _charsets.add(Charset(65, 'ascii', 'ascii_bin', '')) @@ -128,6 +123,7 @@ _charsets.add(Charset(72, 'hp8', 'hp8_bin', '')) _charsets.add(Charset(73, 'keybcs2', 'keybcs2_bin', '')) _charsets.add(Charset(74, 'koi8r', 'koi8r_bin', '')) _charsets.add(Charset(75, 'koi8u', 'koi8u_bin', '')) +_charsets.add(Charset(76, 'utf8', 'utf8_tolower_ci', '')) _charsets.add(Charset(77, 'latin2', 'latin2_bin', '')) _charsets.add(Charset(78, 'latin5', 'latin5_bin', '')) _charsets.add(Charset(79, 'latin7', 'latin7_bin', '')) @@ -141,7 +137,6 @@ _charsets.add(Charset(86, 'gb2312', 'gb2312_bin', '')) _charsets.add(Charset(87, 'gbk', 'gbk_bin', '')) _charsets.add(Charset(88, 'sjis', 'sjis_bin', '')) _charsets.add(Charset(89, 'tis620', 'tis620_bin', '')) -_charsets.add(Charset(90, 'ucs2', 'ucs2_bin', '')) _charsets.add(Charset(91, 'ujis', 'ujis_bin', '')) _charsets.add(Charset(92, 'geostd8', 'geostd8_general_ci', 'Yes')) _charsets.add(Charset(93, 'geostd8', 'geostd8_bin', '')) @@ -151,67 +146,6 @@ _charsets.add(Charset(96, 'cp932', 'cp932_bin', '')) _charsets.add(Charset(97, 'eucjpms', 'eucjpms_japanese_ci', 'Yes')) _charsets.add(Charset(98, 'eucjpms', 'eucjpms_bin', '')) _charsets.add(Charset(99, 'cp1250', 'cp1250_polish_ci', '')) -_charsets.add(Charset(101, 'utf16', 'utf16_unicode_ci', '')) -_charsets.add(Charset(102, 'utf16', 'utf16_icelandic_ci', '')) -_charsets.add(Charset(103, 'utf16', 'utf16_latvian_ci', '')) -_charsets.add(Charset(104, 'utf16', 'utf16_romanian_ci', '')) -_charsets.add(Charset(105, 'utf16', 'utf16_slovenian_ci', '')) -_charsets.add(Charset(106, 'utf16', 'utf16_polish_ci', '')) -_charsets.add(Charset(107, 'utf16', 'utf16_estonian_ci', '')) -_charsets.add(Charset(108, 'utf16', 'utf16_spanish_ci', '')) -_charsets.add(Charset(109, 'utf16', 'utf16_swedish_ci', '')) -_charsets.add(Charset(110, 'utf16', 'utf16_turkish_ci', '')) -_charsets.add(Charset(111, 'utf16', 'utf16_czech_ci', '')) -_charsets.add(Charset(112, 'utf16', 'utf16_danish_ci', '')) -_charsets.add(Charset(113, 'utf16', 'utf16_lithuanian_ci', '')) -_charsets.add(Charset(114, 'utf16', 'utf16_slovak_ci', '')) -_charsets.add(Charset(115, 'utf16', 'utf16_spanish2_ci', '')) -_charsets.add(Charset(116, 'utf16', 'utf16_roman_ci', '')) -_charsets.add(Charset(117, 'utf16', 'utf16_persian_ci', '')) -_charsets.add(Charset(118, 'utf16', 'utf16_esperanto_ci', '')) -_charsets.add(Charset(119, 'utf16', 'utf16_hungarian_ci', '')) -_charsets.add(Charset(120, 'utf16', 'utf16_sinhala_ci', '')) -_charsets.add(Charset(128, 'ucs2', 'ucs2_unicode_ci', '')) -_charsets.add(Charset(129, 'ucs2', 'ucs2_icelandic_ci', '')) -_charsets.add(Charset(130, 'ucs2', 'ucs2_latvian_ci', '')) -_charsets.add(Charset(131, 'ucs2', 'ucs2_romanian_ci', '')) -_charsets.add(Charset(132, 'ucs2', 'ucs2_slovenian_ci', '')) -_charsets.add(Charset(133, 'ucs2', 'ucs2_polish_ci', '')) -_charsets.add(Charset(134, 'ucs2', 'ucs2_estonian_ci', '')) -_charsets.add(Charset(135, 'ucs2', 'ucs2_spanish_ci', '')) -_charsets.add(Charset(136, 'ucs2', 'ucs2_swedish_ci', '')) -_charsets.add(Charset(137, 'ucs2', 'ucs2_turkish_ci', '')) -_charsets.add(Charset(138, 'ucs2', 'ucs2_czech_ci', '')) -_charsets.add(Charset(139, 'ucs2', 'ucs2_danish_ci', '')) -_charsets.add(Charset(140, 'ucs2', 'ucs2_lithuanian_ci', '')) -_charsets.add(Charset(141, 'ucs2', 'ucs2_slovak_ci', '')) -_charsets.add(Charset(142, 'ucs2', 'ucs2_spanish2_ci', '')) -_charsets.add(Charset(143, 'ucs2', 'ucs2_roman_ci', '')) -_charsets.add(Charset(144, 'ucs2', 'ucs2_persian_ci', '')) -_charsets.add(Charset(145, 'ucs2', 'ucs2_esperanto_ci', '')) -_charsets.add(Charset(146, 'ucs2', 'ucs2_hungarian_ci', '')) -_charsets.add(Charset(147, 'ucs2', 'ucs2_sinhala_ci', '')) -_charsets.add(Charset(159, 'ucs2', 'ucs2_general_mysql500_ci', '')) -_charsets.add(Charset(160, 'utf32', 'utf32_unicode_ci', '')) -_charsets.add(Charset(161, 'utf32', 'utf32_icelandic_ci', '')) -_charsets.add(Charset(162, 'utf32', 'utf32_latvian_ci', '')) -_charsets.add(Charset(163, 'utf32', 'utf32_romanian_ci', '')) -_charsets.add(Charset(164, 'utf32', 'utf32_slovenian_ci', '')) -_charsets.add(Charset(165, 'utf32', 'utf32_polish_ci', '')) -_charsets.add(Charset(166, 'utf32', 'utf32_estonian_ci', '')) -_charsets.add(Charset(167, 'utf32', 'utf32_spanish_ci', '')) -_charsets.add(Charset(168, 'utf32', 'utf32_swedish_ci', '')) -_charsets.add(Charset(169, 'utf32', 'utf32_turkish_ci', '')) -_charsets.add(Charset(170, 'utf32', 'utf32_czech_ci', '')) -_charsets.add(Charset(171, 'utf32', 'utf32_danish_ci', '')) -_charsets.add(Charset(172, 'utf32', 'utf32_lithuanian_ci', '')) -_charsets.add(Charset(173, 'utf32', 'utf32_slovak_ci', '')) -_charsets.add(Charset(174, 'utf32', 'utf32_spanish2_ci', '')) -_charsets.add(Charset(175, 'utf32', 'utf32_roman_ci', '')) -_charsets.add(Charset(176, 'utf32', 'utf32_persian_ci', '')) -_charsets.add(Charset(177, 'utf32', 'utf32_esperanto_ci', '')) -_charsets.add(Charset(178, 'utf32', 'utf32_hungarian_ci', '')) -_charsets.add(Charset(179, 'utf32', 'utf32_sinhala_ci', '')) _charsets.add(Charset(192, 'utf8', 'utf8_unicode_ci', '')) _charsets.add(Charset(193, 'utf8', 'utf8_icelandic_ci', '')) _charsets.add(Charset(194, 'utf8', 'utf8_latvian_ci', '')) @@ -232,6 +166,10 @@ _charsets.add(Charset(208, 'utf8', 'utf8_persian_ci', '')) _charsets.add(Charset(209, 'utf8', 'utf8_esperanto_ci', '')) _charsets.add(Charset(210, 'utf8', 'utf8_hungarian_ci', '')) _charsets.add(Charset(211, 'utf8', 'utf8_sinhala_ci', '')) +_charsets.add(Charset(212, 'utf8', 'utf8_german2_ci', '')) +_charsets.add(Charset(213, 'utf8', 'utf8_croatian_ci', '')) +_charsets.add(Charset(214, 'utf8', 'utf8_unicode_520_ci', '')) +_charsets.add(Charset(215, 'utf8', 'utf8_vietnamese_ci', '')) _charsets.add(Charset(223, 'utf8', 'utf8_general_mysql500_ci', '')) _charsets.add(Charset(224, 'utf8mb4', 'utf8mb4_unicode_ci', '')) _charsets.add(Charset(225, 'utf8mb4', 'utf8mb4_icelandic_ci', '')) @@ -257,14 +195,18 @@ _charsets.add(Charset(244, 'utf8mb4', 'utf8mb4_german2_ci', '')) _charsets.add(Charset(245, 'utf8mb4', 'utf8mb4_croatian_ci', '')) _charsets.add(Charset(246, 'utf8mb4', 'utf8mb4_unicode_520_ci', '')) _charsets.add(Charset(247, 'utf8mb4', 'utf8mb4_vietnamese_ci', '')) - +_charsets.add(Charset(248, 'gb18030', 'gb18030_chinese_ci', 'Yes')) +_charsets.add(Charset(249, 'gb18030', 'gb18030_bin', '')) +_charsets.add(Charset(250, 'gb18030', 'gb18030_unicode_520_ci', '')) +_charsets.add(Charset(255, 'utf8mb4', 'utf8mb4_0900_ai_ci', '')) charset_by_name = _charsets.by_name charset_by_id = _charsets.by_id +#TODO: remove this def charset_to_encoding(name): """Convert MySQL's charset name to Python's codec name""" - if name == 'utf8mb4': + if name in ('utf8mb4', 'utf8mb3'): return 'utf8' return name diff --git a/server/www/packages/packages-linux/x64/pymysql/connections.py b/server/www/packages/packages-linux/x64/pymysql/connections.py index 1e580d2..2e4122b 100644 --- a/server/www/packages/packages-linux/x64/pymysql/connections.py +++ b/server/www/packages/packages-linux/x64/pymysql/connections.py @@ -88,7 +88,7 @@ TEXT_TYPES = { } -DEFAULT_CHARSET = 'utf8mb4' # TODO: change to utf8mb4 +DEFAULT_CHARSET = 'utf8mb4' MAX_PACKET_LEN = 2**24-1 @@ -152,7 +152,6 @@ class Connection(object): (default: 10, min: 1, max: 31536000) :param ssl: A dict of arguments similar to mysql_ssl_set()'s parameters. - For now the capath and cipher arguments are not supported. :param read_default_group: Group to read from in the configuration file. :param compress: Not supported :param named_pipe: Not supported @@ -295,15 +294,15 @@ class Connection(object): self._affected_rows = 0 self.host_info = "Not connected" - #: specified autocommit mode. None means use server default. + # specified autocommit mode. None means use server default. self.autocommit_mode = autocommit if conv is None: conv = converters.conversions # Need for MySQLdb compatibility. - self.encoders = dict([(k, v) for (k, v) in conv.items() if type(k) is not int]) - self.decoders = dict([(k, v) for (k, v) in conv.items() if type(k) is int]) + self.encoders = {k: v for (k, v) in conv.items() if type(k) is not int} + self.decoders = {k: v for (k, v) in conv.items() if type(k) is int} self.sql_mode = sql_mode self.init_command = init_command self.max_allowed_packet = max_allowed_packet @@ -316,10 +315,9 @@ class Connection(object): '_pid': str(os.getpid()), '_client_version': VERSION_STRING, } + if program_name: self._connect_attrs["program_name"] = program_name - elif sys.argv: - self._connect_attrs["program_name"] = sys.argv[0] if defer_connect: self._sock = None @@ -494,6 +492,9 @@ class Connection(object): def __enter__(self): """Context manager that returns a Cursor""" + warnings.warn( + "Context manager API of Connection object is deprecated; Use conn.begin()", + DeprecationWarning) return self.cursor() def __exit__(self, exc, value, traceback): @@ -696,6 +697,10 @@ class Connection(object): raise err.OperationalError( CR.CR_SERVER_LOST, "Lost connection to MySQL server during query (%s)" % (e,)) + except BaseException: + # Don't convert unknown exception to MySQLError. + self._force_close() + raise if len(data) < num_bytes: self._force_close() raise err.OperationalError( @@ -804,7 +809,11 @@ class Connection(object): authresp = b'' plugin_name = None - if self._auth_plugin_name in ('', 'mysql_native_password'): + if self._auth_plugin_name == '': + plugin_name = b'' + authresp = _auth.scramble_native_password(self.password, self.salt) + elif self._auth_plugin_name == 'mysql_native_password': + plugin_name = b'mysql_native_password' authresp = _auth.scramble_native_password(self.password, self.salt) elif self._auth_plugin_name == 'caching_sha2_password': plugin_name = b'caching_sha2_password' @@ -842,9 +851,9 @@ class Connection(object): if self.server_capabilities & CLIENT.CONNECT_ATTRS: connect_attrs = b'' for k, v in self._connect_attrs.items(): - k = k.encode('utf8') + k = k.encode('utf-8') connect_attrs += struct.pack('B', len(k)) + k - v = v.encode('utf8') + v = v.encode('utf-8') connect_attrs += struct.pack('B', len(v)) + v data += struct.pack('B', len(connect_attrs)) + connect_attrs diff --git a/server/www/packages/packages-linux/x64/pymysql/converters.py b/server/www/packages/packages-linux/x64/pymysql/converters.py index bf1db9d..ce2be06 100644 --- a/server/www/packages/packages-linux/x64/pymysql/converters.py +++ b/server/www/packages/packages-linux/x64/pymysql/converters.py @@ -354,21 +354,6 @@ def through(x): convert_bit = through -def convert_characters(connection, field, data): - field_charset = charset_by_id(field.charsetnr).name - encoding = charset_to_encoding(field_charset) - if field.flags & FLAG.SET: - return convert_set(data.decode(encoding)) - if field.flags & FLAG.BINARY: - return data - - if connection.use_unicode: - data = data.decode(encoding) - elif connection.charset != field_charset: - data = data.decode(encoding) - data = data.encode(connection.encoding) - return data - encoders = { bool: escape_bool, int: escape_int, diff --git a/server/www/packages/packages-linux/x64/pymysql/cursors.py b/server/www/packages/packages-linux/x64/pymysql/cursors.py index cc16998..a6d645d 100644 --- a/server/www/packages/packages-linux/x64/pymysql/cursors.py +++ b/server/www/packages/packages-linux/x64/pymysql/cursors.py @@ -122,9 +122,9 @@ class Cursor(object): return tuple(conn.literal(arg) for arg in args) elif isinstance(args, dict): if PY2: - args = dict((ensure_bytes(key), ensure_bytes(val)) for - (key, val) in args.items()) - return dict((key, conn.literal(val)) for (key, val) in args.items()) + args = {ensure_bytes(key): ensure_bytes(val) for + (key, val) in args.items()} + return {key: conn.literal(val) for (key, val) in args.items()} else: # If it's not a dictionary let's try escaping it anyways. # Worst case it will throw a Value error diff --git a/server/www/packages/packages-linux/x64/pymysql/util.py b/server/www/packages/packages-linux/x64/pymysql/util.py index 3e82ac7..04683f8 100644 --- a/server/www/packages/packages-linux/x64/pymysql/util.py +++ b/server/www/packages/packages-linux/x64/pymysql/util.py @@ -11,12 +11,3 @@ def byte2int(b): def int2byte(i): return struct.pack("!B", i) - -def join_bytes(bs): - if len(bs) == 0: - return "" - else: - rv = bs[0] - for b in bs[1:]: - rv += b - return rv diff --git a/server/www/packages/packages-linux/x64/qrcode/main.py b/server/www/packages/packages-linux/x64/qrcode/main.py index 1e164f1..e46a9b9 100644 --- a/server/www/packages/packages-linux/x64/qrcode/main.py +++ b/server/www/packages/packages-linux/x64/qrcode/main.py @@ -33,6 +33,7 @@ def _check_mask_pattern(mask_pattern): raise ValueError( "Mask pattern should be in range(8) (got %s)" % mask_pattern) + class QRCode: def __init__(self, version=None, diff --git a/server/www/packages/packages-linux/x64/qrcode/release.py b/server/www/packages/packages-linux/x64/qrcode/release.py index abbabb4..4cc4c19 100644 --- a/server/www/packages/packages-linux/x64/qrcode/release.py +++ b/server/www/packages/packages-linux/x64/qrcode/release.py @@ -12,7 +12,6 @@ def update_manpage(data): Update the version in the manpage document. """ if data['name'] != 'qrcode': - print('no qrcode') return base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) diff --git a/server/www/packages/packages-linux/x64/qrcode/util.py b/server/www/packages/packages-linux/x64/qrcode/util.py index a9652f7..231b85e 100644 --- a/server/www/packages/packages-linux/x64/qrcode/util.py +++ b/server/www/packages/packages-linux/x64/qrcode/util.py @@ -33,7 +33,7 @@ MODE_SIZE_LARGE = { } ALPHA_NUM = six.b('0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ $%*+-./:') -RE_ALPHA_NUM = re.compile(six.b('^[') + re.escape(ALPHA_NUM) + six.b(']*\Z')) +RE_ALPHA_NUM = re.compile(six.b('^[') + re.escape(ALPHA_NUM) + six.b(r']*\Z')) # The number of bits for numeric delimited data lengths. NUMBER_LENGTH = {3: 10, 2: 7, 1: 4} @@ -344,12 +344,17 @@ def optimal_data_chunks(data, minimum=4): :param minimum: The minimum number of bytes in a row to split as a chunk. """ data = to_bytestring(data) - re_repeat = ( - six.b('{') + six.text_type(minimum).encode('ascii') + six.b(',}')) - num_pattern = re.compile(six.b('\d') + re_repeat) + num_pattern = six.b(r'\d') + alpha_pattern = six.b('[') + re.escape(ALPHA_NUM) + six.b(']') + if len(data) <= minimum: + num_pattern = re.compile(six.b('^') + num_pattern + six.b('+$')) + alpha_pattern = re.compile(six.b('^') + alpha_pattern + six.b('+$')) + else: + re_repeat = ( + six.b('{') + six.text_type(minimum).encode('ascii') + six.b(',}')) + num_pattern = re.compile(num_pattern + re_repeat) + alpha_pattern = re.compile(alpha_pattern + re_repeat) num_bits = _optimal_split(data, num_pattern) - alpha_pattern = re.compile( - six.b('[') + re.escape(ALPHA_NUM) + six.b(']') + re_repeat) for is_num, chunk in num_bits: if is_num: yield QRData(chunk, mode=MODE_NUMBER, check_data=False) diff --git a/server/www/packages/packages-linux/x64/six.py b/server/www/packages/packages-linux/x64/six.py index 6bf4fd3..357e624 100644 --- a/server/www/packages/packages-linux/x64/six.py +++ b/server/www/packages/packages-linux/x64/six.py @@ -1,4 +1,4 @@ -# Copyright (c) 2010-2017 Benjamin Peterson +# Copyright (c) 2010-2019 Benjamin Peterson # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -29,7 +29,7 @@ import sys import types __author__ = "Benjamin Peterson " -__version__ = "1.11.0" +__version__ = "1.13.0" # Useful for very coarse version differentiation. @@ -255,8 +255,10 @@ _moved_attributes = [ MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), MovedModule("builtins", "__builtin__"), MovedModule("configparser", "ConfigParser"), + MovedModule("collections_abc", "collections", "collections.abc" if sys.version_info >= (3, 3) else "collections"), MovedModule("copyreg", "copy_reg"), MovedModule("dbm_gnu", "gdbm", "dbm.gnu"), + MovedModule("dbm_ndbm", "dbm", "dbm.ndbm"), MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"), MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), MovedModule("http_cookies", "Cookie", "http.cookies"), @@ -637,6 +639,7 @@ if PY3: import io StringIO = io.StringIO BytesIO = io.BytesIO + del io _assertCountEqual = "assertCountEqual" if sys.version_info[1] <= 1: _assertRaisesRegex = "assertRaisesRegexp" @@ -824,7 +827,15 @@ def with_metaclass(meta, *bases): class metaclass(type): def __new__(cls, name, this_bases, d): - return meta(name, bases, d) + if sys.version_info[:2] >= (3, 7): + # This version introduced PEP 560 that requires a bit + # of extra care (we mimic what is done by __build_class__). + resolved_bases = types.resolve_bases(bases) + if resolved_bases is not bases: + d['__orig_bases__'] = bases + else: + resolved_bases = bases + return meta(name, resolved_bases, d) @classmethod def __prepare__(cls, name, this_bases): @@ -844,10 +855,71 @@ def add_metaclass(metaclass): orig_vars.pop(slots_var) orig_vars.pop('__dict__', None) orig_vars.pop('__weakref__', None) + if hasattr(cls, '__qualname__'): + orig_vars['__qualname__'] = cls.__qualname__ return metaclass(cls.__name__, cls.__bases__, orig_vars) return wrapper +def ensure_binary(s, encoding='utf-8', errors='strict'): + """Coerce **s** to six.binary_type. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> encoded to `bytes` + - `bytes` -> `bytes` + """ + if isinstance(s, text_type): + return s.encode(encoding, errors) + elif isinstance(s, binary_type): + return s + else: + raise TypeError("not expecting type '%s'" % type(s)) + + +def ensure_str(s, encoding='utf-8', errors='strict'): + """Coerce *s* to `str`. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if not isinstance(s, (text_type, binary_type)): + raise TypeError("not expecting type '%s'" % type(s)) + if PY2 and isinstance(s, text_type): + s = s.encode(encoding, errors) + elif PY3 and isinstance(s, binary_type): + s = s.decode(encoding, errors) + return s + + +def ensure_text(s, encoding='utf-8', errors='strict'): + """Coerce *s* to six.text_type. + + For Python 2: + - `unicode` -> `unicode` + - `str` -> `unicode` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if isinstance(s, binary_type): + return s.decode(encoding, errors) + elif isinstance(s, text_type): + return s + else: + raise TypeError("not expecting type '%s'" % type(s)) + + + def python_2_unicode_compatible(klass): """ A decorator that defines __unicode__ and __str__ methods under Python 2. diff --git a/server/www/packages/packages-linux/x64/tornado/__init__.py b/server/www/packages/packages-linux/x64/tornado/__init__.py index b269cf7..a1094a6 100644 --- a/server/www/packages/packages-linux/x64/tornado/__init__.py +++ b/server/www/packages/packages-linux/x64/tornado/__init__.py @@ -15,8 +15,6 @@ """The Tornado web server and tools.""" -from __future__ import absolute_import, division, print_function - # version is a human-readable version number. # version_info is a four-tuple for programmatic comparison. The first @@ -24,5 +22,5 @@ from __future__ import absolute_import, division, print_function # is zero for an official release, positive for a development branch, # or negative for a release candidate or beta (after the base version # number has been incremented) -version = "5.1.1" -version_info = (5, 1, 1, 0) +version = "6.0.3" +version_info = (6, 0, 3, 0) diff --git a/server/www/packages/packages-linux/x64/tornado/_locale_data.py b/server/www/packages/packages-linux/x64/tornado/_locale_data.py index a2c5039..91416d9 100644 --- a/server/www/packages/packages-linux/x64/tornado/_locale_data.py +++ b/server/www/packages/packages-linux/x64/tornado/_locale_data.py @@ -16,8 +16,6 @@ """Data used by the tornado.locale module.""" -from __future__ import absolute_import, division, print_function - LOCALE_NAMES = { "af_ZA": {"name_en": u"Afrikaans", "name": u"Afrikaans"}, "am_ET": {"name_en": u"Amharic", "name": u"釆犪垱釄姏"}, diff --git a/server/www/packages/packages-linux/x64/tornado/auth.py b/server/www/packages/packages-linux/x64/tornado/auth.py index b79ad14..db6d290 100644 --- a/server/www/packages/packages-linux/x64/tornado/auth.py +++ b/server/www/packages/packages-linux/x64/tornado/auth.py @@ -54,93 +54,29 @@ Example usage for Google OAuth: .. testoutput:: :hide: - -.. versionchanged:: 4.0 - All of the callback interfaces in this module are now guaranteed - to run their callback with an argument of ``None`` on error. - Previously some functions would do this while others would simply - terminate the request on their own. This change also ensures that - errors are more consistently reported through the ``Future`` interfaces. """ -from __future__ import absolute_import, division, print_function - import base64 import binascii -import functools import hashlib import hmac import time +import urllib.parse import uuid -import warnings -from tornado.concurrent import (Future, _non_deprecated_return_future, - future_set_exc_info, chain_future, - future_set_result_unless_cancelled) -from tornado import gen from tornado import httpclient from tornado import escape from tornado.httputil import url_concat -from tornado.log import gen_log -from tornado.stack_context import ExceptionStackContext, wrap -from tornado.util import unicode_type, ArgReplacer, PY3 +from tornado.util import unicode_type +from tornado.web import RequestHandler -if PY3: - import urllib.parse as urlparse - import urllib.parse as urllib_parse - long = int -else: - import urlparse - import urllib as urllib_parse +from typing import List, Any, Dict, cast, Iterable, Union, Optional class AuthError(Exception): pass -def _auth_future_to_callback(callback, future): - try: - result = future.result() - except AuthError as e: - gen_log.warning(str(e)) - result = None - callback(result) - - -def _auth_return_future(f): - """Similar to tornado.concurrent.return_future, but uses the auth - module's legacy callback interface. - - Note that when using this decorator the ``callback`` parameter - inside the function will actually be a future. - - .. deprecated:: 5.1 - Will be removed in 6.0. - """ - replacer = ArgReplacer(f, 'callback') - - @functools.wraps(f) - def wrapper(*args, **kwargs): - future = Future() - callback, args, kwargs = replacer.replace(future, args, kwargs) - if callback is not None: - warnings.warn("callback arguments are deprecated, use the returned Future instead", - DeprecationWarning) - future.add_done_callback( - wrap(functools.partial(_auth_future_to_callback, callback))) - - def handle_exception(typ, value, tb): - if future.done(): - return False - else: - future_set_exc_info(future, (typ, value, tb)) - return True - with ExceptionStackContext(handle_exception, delay_warning=True): - f(*args, **kwargs) - return future - return wrapper - - class OpenIdMixin(object): """Abstract implementation of OpenID and Attribute Exchange. @@ -148,10 +84,12 @@ class OpenIdMixin(object): * ``_OPENID_ENDPOINT``: the identity provider's URI. """ - @_non_deprecated_return_future - def authenticate_redirect(self, callback_uri=None, - ax_attrs=["name", "email", "language", "username"], - callback=None): + + def authenticate_redirect( + self, + callback_uri: str = None, + ax_attrs: List[str] = ["name", "email", "language", "username"], + ) -> None: """Redirects to the authentication URL for this service. After authentication, the service will redirect back to the given @@ -162,24 +100,22 @@ class OpenIdMixin(object): all those attributes for your app, you can request fewer with the ax_attrs keyword argument. - .. versionchanged:: 3.1 - Returns a `.Future` and takes an optional callback. These are - not strictly necessary as this method is synchronous, - but they are supplied for consistency with - `OAuthMixin.authorize_redirect`. + .. versionchanged:: 6.0 - .. deprecated:: 5.1 - - The ``callback`` argument and returned awaitable will be removed - in Tornado 6.0; this will be an ordinary synchronous function. + The ``callback`` argument was removed and this method no + longer returns an awaitable object. It is now an ordinary + synchronous function. """ - callback_uri = callback_uri or self.request.uri + handler = cast(RequestHandler, self) + callback_uri = callback_uri or handler.request.uri + assert callback_uri is not None args = self._openid_args(callback_uri, ax_attrs=ax_attrs) - self.redirect(self._OPENID_ENDPOINT + "?" + urllib_parse.urlencode(args)) - callback() + endpoint = self._OPENID_ENDPOINT # type: ignore + handler.redirect(endpoint + "?" + urllib.parse.urlencode(args)) - @_auth_return_future - def get_authenticated_user(self, callback, http_client=None): + async def get_authenticated_user( + self, http_client: httpclient.AsyncHTTPClient = None + ) -> Dict[str, Any]: """Fetches the authenticated user data upon redirect. This method should be called by the handler that receives the @@ -190,51 +126,57 @@ class OpenIdMixin(object): The result of this method will generally be used to set a cookie. - .. deprecated:: 5.1 + .. versionchanged:: 6.0 - The ``callback`` argument is deprecated and will be removed in 6.0. - Use the returned awaitable object instead. + The ``callback`` argument was removed. Use the returned + awaitable object instead. """ + handler = cast(RequestHandler, self) # Verify the OpenID response via direct request to the OP - args = dict((k, v[-1]) for k, v in self.request.arguments.items()) + args = dict( + (k, v[-1]) for k, v in handler.request.arguments.items() + ) # type: Dict[str, Union[str, bytes]] args["openid.mode"] = u"check_authentication" - url = self._OPENID_ENDPOINT + url = self._OPENID_ENDPOINT # type: ignore if http_client is None: http_client = self.get_auth_http_client() - fut = http_client.fetch(url, method="POST", body=urllib_parse.urlencode(args)) - fut.add_done_callback(wrap(functools.partial( - self._on_authentication_verified, callback))) + resp = await http_client.fetch( + url, method="POST", body=urllib.parse.urlencode(args) + ) + return self._on_authentication_verified(resp) - def _openid_args(self, callback_uri, ax_attrs=[], oauth_scope=None): - url = urlparse.urljoin(self.request.full_url(), callback_uri) + def _openid_args( + self, callback_uri: str, ax_attrs: Iterable[str] = [], oauth_scope: str = None + ) -> Dict[str, str]: + handler = cast(RequestHandler, self) + url = urllib.parse.urljoin(handler.request.full_url(), callback_uri) args = { "openid.ns": "http://specs.openid.net/auth/2.0", - "openid.claimed_id": - "http://specs.openid.net/auth/2.0/identifier_select", - "openid.identity": - "http://specs.openid.net/auth/2.0/identifier_select", + "openid.claimed_id": "http://specs.openid.net/auth/2.0/identifier_select", + "openid.identity": "http://specs.openid.net/auth/2.0/identifier_select", "openid.return_to": url, - "openid.realm": urlparse.urljoin(url, '/'), + "openid.realm": urllib.parse.urljoin(url, "/"), "openid.mode": "checkid_setup", } if ax_attrs: - args.update({ - "openid.ns.ax": "http://openid.net/srv/ax/1.0", - "openid.ax.mode": "fetch_request", - }) + args.update( + { + "openid.ns.ax": "http://openid.net/srv/ax/1.0", + "openid.ax.mode": "fetch_request", + } + ) ax_attrs = set(ax_attrs) - required = [] + required = [] # type: List[str] if "name" in ax_attrs: ax_attrs -= set(["name", "firstname", "fullname", "lastname"]) required += ["firstname", "fullname", "lastname"] - args.update({ - "openid.ax.type.firstname": - "http://axschema.org/namePerson/first", - "openid.ax.type.fullname": - "http://axschema.org/namePerson", - "openid.ax.type.lastname": - "http://axschema.org/namePerson/last", - }) + args.update( + { + "openid.ax.type.firstname": "http://axschema.org/namePerson/first", + "openid.ax.type.fullname": "http://axschema.org/namePerson", + "openid.ax.type.lastname": "http://axschema.org/namePerson/last", + } + ) known_attrs = { "email": "http://axschema.org/contact/email", "language": "http://axschema.org/pref/language", @@ -245,47 +187,45 @@ class OpenIdMixin(object): required.append(name) args["openid.ax.required"] = ",".join(required) if oauth_scope: - args.update({ - "openid.ns.oauth": - "http://specs.openid.net/extensions/oauth/1.0", - "openid.oauth.consumer": self.request.host.split(":")[0], - "openid.oauth.scope": oauth_scope, - }) + args.update( + { + "openid.ns.oauth": "http://specs.openid.net/extensions/oauth/1.0", + "openid.oauth.consumer": handler.request.host.split(":")[0], + "openid.oauth.scope": oauth_scope, + } + ) return args - def _on_authentication_verified(self, future, response_fut): - try: - response = response_fut.result() - except Exception as e: - future.set_exception(AuthError( - "Error response %s" % e)) - return + def _on_authentication_verified( + self, response: httpclient.HTTPResponse + ) -> Dict[str, Any]: + handler = cast(RequestHandler, self) if b"is_valid:true" not in response.body: - future.set_exception(AuthError( - "Invalid OpenID response: %s" % response.body)) - return + raise AuthError("Invalid OpenID response: %s" % response.body) # Make sure we got back at least an email from attribute exchange ax_ns = None - for name in self.request.arguments: - if name.startswith("openid.ns.") and \ - self.get_argument(name) == u"http://openid.net/srv/ax/1.0": - ax_ns = name[10:] + for key in handler.request.arguments: + if ( + key.startswith("openid.ns.") + and handler.get_argument(key) == u"http://openid.net/srv/ax/1.0" + ): + ax_ns = key[10:] break - def get_ax_arg(uri): + def get_ax_arg(uri: str) -> str: if not ax_ns: return u"" prefix = "openid." + ax_ns + ".type." ax_name = None - for name in self.request.arguments.keys(): - if self.get_argument(name) == uri and name.startswith(prefix): - part = name[len(prefix):] + for name in handler.request.arguments.keys(): + if handler.get_argument(name) == uri and name.startswith(prefix): + part = name[len(prefix) :] ax_name = "openid." + ax_ns + ".value." + part break if not ax_name: return u"" - return self.get_argument(ax_name, u"") + return handler.get_argument(ax_name, u"") email = get_ax_arg("http://axschema.org/contact/email") name = get_ax_arg("http://axschema.org/namePerson") @@ -313,12 +253,12 @@ class OpenIdMixin(object): user["locale"] = locale if username: user["username"] = username - claimed_id = self.get_argument("openid.claimed_id", None) + claimed_id = handler.get_argument("openid.claimed_id", None) if claimed_id: user["claimed_id"] = claimed_id - future_set_result_unless_cancelled(future, user) + return user - def get_auth_http_client(self): + def get_auth_http_client(self) -> httpclient.AsyncHTTPClient: """Returns the `.AsyncHTTPClient` instance to be used for auth requests. May be overridden by subclasses to use an HTTP client other than @@ -343,9 +283,13 @@ class OAuthMixin(object): Subclasses must also override the `_oauth_get_user_future` and `_oauth_consumer_token` methods. """ - @_non_deprecated_return_future - def authorize_redirect(self, callback_uri=None, extra_params=None, - http_client=None, callback=None): + + async def authorize_redirect( + self, + callback_uri: str = None, + extra_params: Dict[str, Any] = None, + http_client: httpclient.AsyncHTTPClient = None, + ) -> None: """Redirects the user to obtain OAuth authorization for this service. The ``callback_uri`` may be omitted if you have previously @@ -367,35 +311,31 @@ class OAuthMixin(object): Now returns a `.Future` and takes an optional callback, for compatibility with `.gen.coroutine`. - .. deprecated:: 5.1 + .. versionchanged:: 6.0 - The ``callback`` argument is deprecated and will be removed in 6.0. - Use the returned awaitable object instead. + The ``callback`` argument was removed. Use the returned + awaitable object instead. """ if callback_uri and getattr(self, "_OAUTH_NO_CALLBACKS", False): raise Exception("This service does not support oauth_callback") if http_client is None: http_client = self.get_auth_http_client() + assert http_client is not None if getattr(self, "_OAUTH_VERSION", "1.0a") == "1.0a": - fut = http_client.fetch( - self._oauth_request_token_url(callback_uri=callback_uri, - extra_params=extra_params)) - fut.add_done_callback(wrap(functools.partial( - self._on_request_token, - self._OAUTH_AUTHORIZE_URL, - callback_uri, - callback))) + response = await http_client.fetch( + self._oauth_request_token_url( + callback_uri=callback_uri, extra_params=extra_params + ) + ) else: - fut = http_client.fetch(self._oauth_request_token_url()) - fut.add_done_callback( - wrap(functools.partial( - self._on_request_token, self._OAUTH_AUTHORIZE_URL, - callback_uri, - callback))) + response = await http_client.fetch(self._oauth_request_token_url()) + url = self._OAUTH_AUTHORIZE_URL # type: ignore + self._on_request_token(url, callback_uri, response) - @_auth_return_future - def get_authenticated_user(self, callback, http_client=None): + async def get_authenticated_user( + self, http_client: httpclient.AsyncHTTPClient = None + ) -> Dict[str, Any]: """Gets the OAuth authorized user and access token. This method should be called from the handler for your @@ -406,37 +346,45 @@ class OAuthMixin(object): also contain other fields such as ``name``, depending on the service used. - .. deprecated:: 5.1 + .. versionchanged:: 6.0 - The ``callback`` argument is deprecated and will be removed in 6.0. - Use the returned awaitable object instead. + The ``callback`` argument was removed. Use the returned + awaitable object instead. """ - future = callback - request_key = escape.utf8(self.get_argument("oauth_token")) - oauth_verifier = self.get_argument("oauth_verifier", None) - request_cookie = self.get_cookie("_oauth_request_token") + handler = cast(RequestHandler, self) + request_key = escape.utf8(handler.get_argument("oauth_token")) + oauth_verifier = handler.get_argument("oauth_verifier", None) + request_cookie = handler.get_cookie("_oauth_request_token") if not request_cookie: - future.set_exception(AuthError( - "Missing OAuth request token cookie")) - return - self.clear_cookie("_oauth_request_token") + raise AuthError("Missing OAuth request token cookie") + handler.clear_cookie("_oauth_request_token") cookie_key, cookie_secret = [ - base64.b64decode(escape.utf8(i)) for i in request_cookie.split("|")] + base64.b64decode(escape.utf8(i)) for i in request_cookie.split("|") + ] if cookie_key != request_key: - future.set_exception(AuthError( - "Request token does not match cookie")) - return - token = dict(key=cookie_key, secret=cookie_secret) + raise AuthError("Request token does not match cookie") + token = dict( + key=cookie_key, secret=cookie_secret + ) # type: Dict[str, Union[str, bytes]] if oauth_verifier: token["verifier"] = oauth_verifier if http_client is None: http_client = self.get_auth_http_client() - fut = http_client.fetch(self._oauth_access_token_url(token)) - fut.add_done_callback(wrap(functools.partial(self._on_access_token, callback))) + assert http_client is not None + response = await http_client.fetch(self._oauth_access_token_url(token)) + access_token = _oauth_parse_response(response.body) + user = await self._oauth_get_user_future(access_token) + if not user: + raise AuthError("Error getting user") + user["access_token"] = access_token + return user - def _oauth_request_token_url(self, callback_uri=None, extra_params=None): + def _oauth_request_token_url( + self, callback_uri: str = None, extra_params: Dict[str, Any] = None + ) -> str: + handler = cast(RequestHandler, self) consumer_token = self._oauth_consumer_token() - url = self._OAUTH_REQUEST_TOKEN_URL + url = self._OAUTH_REQUEST_TOKEN_URL # type: ignore args = dict( oauth_consumer_key=escape.to_basestring(consumer_token["key"]), oauth_signature_method="HMAC-SHA1", @@ -448,8 +396,9 @@ class OAuthMixin(object): if callback_uri == "oob": args["oauth_callback"] = "oob" elif callback_uri: - args["oauth_callback"] = urlparse.urljoin( - self.request.full_url(), callback_uri) + args["oauth_callback"] = urllib.parse.urljoin( + handler.request.full_url(), callback_uri + ) if extra_params: args.update(extra_params) signature = _oauth10a_signature(consumer_token, "GET", url, args) @@ -457,32 +406,35 @@ class OAuthMixin(object): signature = _oauth_signature(consumer_token, "GET", url, args) args["oauth_signature"] = signature - return url + "?" + urllib_parse.urlencode(args) + return url + "?" + urllib.parse.urlencode(args) - def _on_request_token(self, authorize_url, callback_uri, callback, - response_fut): - try: - response = response_fut.result() - except Exception as e: - raise Exception("Could not get request token: %s" % e) + def _on_request_token( + self, + authorize_url: str, + callback_uri: Optional[str], + response: httpclient.HTTPResponse, + ) -> None: + handler = cast(RequestHandler, self) request_token = _oauth_parse_response(response.body) - data = (base64.b64encode(escape.utf8(request_token["key"])) + b"|" + - base64.b64encode(escape.utf8(request_token["secret"]))) - self.set_cookie("_oauth_request_token", data) + data = ( + base64.b64encode(escape.utf8(request_token["key"])) + + b"|" + + base64.b64encode(escape.utf8(request_token["secret"])) + ) + handler.set_cookie("_oauth_request_token", data) args = dict(oauth_token=request_token["key"]) if callback_uri == "oob": - self.finish(authorize_url + "?" + urllib_parse.urlencode(args)) - callback() + handler.finish(authorize_url + "?" + urllib.parse.urlencode(args)) return elif callback_uri: - args["oauth_callback"] = urlparse.urljoin( - self.request.full_url(), callback_uri) - self.redirect(authorize_url + "?" + urllib_parse.urlencode(args)) - callback() + args["oauth_callback"] = urllib.parse.urljoin( + handler.request.full_url(), callback_uri + ) + handler.redirect(authorize_url + "?" + urllib.parse.urlencode(args)) - def _oauth_access_token_url(self, request_token): + def _oauth_access_token_url(self, request_token: Dict[str, Any]) -> str: consumer_token = self._oauth_consumer_token() - url = self._OAUTH_ACCESS_TOKEN_URL + url = self._OAUTH_ACCESS_TOKEN_URL # type: ignore args = dict( oauth_consumer_key=escape.to_basestring(consumer_token["key"]), oauth_token=escape.to_basestring(request_token["key"]), @@ -495,41 +447,31 @@ class OAuthMixin(object): args["oauth_verifier"] = request_token["verifier"] if getattr(self, "_OAUTH_VERSION", "1.0a") == "1.0a": - signature = _oauth10a_signature(consumer_token, "GET", url, args, - request_token) + signature = _oauth10a_signature( + consumer_token, "GET", url, args, request_token + ) else: - signature = _oauth_signature(consumer_token, "GET", url, args, - request_token) + signature = _oauth_signature( + consumer_token, "GET", url, args, request_token + ) args["oauth_signature"] = signature - return url + "?" + urllib_parse.urlencode(args) + return url + "?" + urllib.parse.urlencode(args) - def _on_access_token(self, future, response_fut): - try: - response = response_fut.result() - except Exception: - future.set_exception(AuthError("Could not fetch access token")) - return - - access_token = _oauth_parse_response(response.body) - fut = self._oauth_get_user_future(access_token) - fut = gen.convert_yielded(fut) - fut.add_done_callback( - wrap(functools.partial(self._on_oauth_get_user, access_token, future))) - - def _oauth_consumer_token(self): + def _oauth_consumer_token(self) -> Dict[str, Any]: """Subclasses must override this to return their OAuth consumer keys. The return value should be a `dict` with keys ``key`` and ``secret``. """ raise NotImplementedError() - @_non_deprecated_return_future - def _oauth_get_user_future(self, access_token, callback): + async def _oauth_get_user_future( + self, access_token: Dict[str, Any] + ) -> Dict[str, Any]: """Subclasses must override this to get basic information about the user. - Should return a `.Future` whose result is a dictionary + Should be a coroutine whose result is a dictionary containing information about the user, which may have been retrieved by using ``access_token`` to make a request to the service. @@ -537,40 +479,23 @@ class OAuthMixin(object): The access token will be added to the returned dictionary to make the result of `get_authenticated_user`. - For backwards compatibility, the callback-based ``_oauth_get_user`` - method is also supported. - .. versionchanged:: 5.1 Subclasses may also define this method with ``async def``. - .. deprecated:: 5.1 + .. versionchanged:: 6.0 - The ``_oauth_get_user`` fallback is deprecated and support for it - will be removed in 6.0. + A synchronous fallback to ``_oauth_get_user`` was removed. """ - warnings.warn("_oauth_get_user is deprecated, override _oauth_get_user_future instead", - DeprecationWarning) - # By default, call the old-style _oauth_get_user, but new code - # should override this method instead. - self._oauth_get_user(access_token, callback) - - def _oauth_get_user(self, access_token, callback): raise NotImplementedError() - def _on_oauth_get_user(self, access_token, future, user_future): - if user_future.exception() is not None: - future.set_exception(user_future.exception()) - return - user = user_future.result() - if not user: - future.set_exception(AuthError("Error getting user")) - return - user["access_token"] = access_token - future_set_result_unless_cancelled(future, user) - - def _oauth_request_parameters(self, url, access_token, parameters={}, - method="GET"): + def _oauth_request_parameters( + self, + url: str, + access_token: Dict[str, Any], + parameters: Dict[str, Any] = {}, + method: str = "GET", + ) -> Dict[str, Any]: """Returns the OAuth parameters as a dict for the given request. parameters should include all POST arguments and query string arguments @@ -589,15 +514,17 @@ class OAuthMixin(object): args.update(base_args) args.update(parameters) if getattr(self, "_OAUTH_VERSION", "1.0a") == "1.0a": - signature = _oauth10a_signature(consumer_token, method, url, args, - access_token) + signature = _oauth10a_signature( + consumer_token, method, url, args, access_token + ) else: - signature = _oauth_signature(consumer_token, method, url, args, - access_token) + signature = _oauth_signature( + consumer_token, method, url, args, access_token + ) base_args["oauth_signature"] = escape.to_basestring(signature) return base_args - def get_auth_http_client(self): + def get_auth_http_client(self) -> httpclient.AsyncHTTPClient: """Returns the `.AsyncHTTPClient` instance to be used for auth requests. May be overridden by subclasses to use an HTTP client other than @@ -617,10 +544,16 @@ class OAuth2Mixin(object): * ``_OAUTH_AUTHORIZE_URL``: The service's authorization url. * ``_OAUTH_ACCESS_TOKEN_URL``: The service's access token url. """ - @_non_deprecated_return_future - def authorize_redirect(self, redirect_uri=None, client_id=None, - client_secret=None, extra_params=None, - callback=None, scope=None, response_type="code"): + + def authorize_redirect( + self, + redirect_uri: str = None, + client_id: str = None, + client_secret: str = None, + extra_params: Dict[str, Any] = None, + scope: str = None, + response_type: str = "code", + ) -> None: """Redirects the user to obtain OAuth authorization for this service. Some providers require that you register a redirect URL with @@ -629,47 +562,53 @@ class OAuth2Mixin(object): ``get_authenticated_user`` in the handler for your redirect URL to complete the authorization process. - .. versionchanged:: 3.1 - Returns a `.Future` and takes an optional callback. These are - not strictly necessary as this method is synchronous, - but they are supplied for consistency with - `OAuthMixin.authorize_redirect`. + .. versionchanged:: 6.0 - .. deprecated:: 5.1 - - The ``callback`` argument and returned awaitable will be removed - in Tornado 6.0; this will be an ordinary synchronous function. + The ``callback`` argument and returned awaitable were removed; + this is now an ordinary synchronous function. """ - args = { - "redirect_uri": redirect_uri, - "client_id": client_id, - "response_type": response_type - } + handler = cast(RequestHandler, self) + args = {"response_type": response_type} + if redirect_uri is not None: + args["redirect_uri"] = redirect_uri + if client_id is not None: + args["client_id"] = client_id if extra_params: args.update(extra_params) if scope: - args['scope'] = ' '.join(scope) - self.redirect( - url_concat(self._OAUTH_AUTHORIZE_URL, args)) - callback() + args["scope"] = " ".join(scope) + url = self._OAUTH_AUTHORIZE_URL # type: ignore + handler.redirect(url_concat(url, args)) - def _oauth_request_token_url(self, redirect_uri=None, client_id=None, - client_secret=None, code=None, - extra_params=None): - url = self._OAUTH_ACCESS_TOKEN_URL - args = dict( - redirect_uri=redirect_uri, - code=code, - client_id=client_id, - client_secret=client_secret, - ) + def _oauth_request_token_url( + self, + redirect_uri: str = None, + client_id: str = None, + client_secret: str = None, + code: str = None, + extra_params: Dict[str, Any] = None, + ) -> str: + url = self._OAUTH_ACCESS_TOKEN_URL # type: ignore + args = {} # type: Dict[str, str] + if redirect_uri is not None: + args["redirect_uri"] = redirect_uri + if code is not None: + args["code"] = code + if client_id is not None: + args["client_id"] = client_id + if client_secret is not None: + args["client_secret"] = client_secret if extra_params: args.update(extra_params) return url_concat(url, args) - @_auth_return_future - def oauth2_request(self, url, callback, access_token=None, - post_args=None, **args): + async def oauth2_request( + self, + url: str, + access_token: str = None, + post_args: Dict[str, Any] = None, + **args: Any + ) -> Any: """Fetches the given URL auth an OAuth2 access token. If the request is a POST, ``post_args`` should be provided. Query @@ -699,10 +638,9 @@ class OAuth2Mixin(object): .. versionadded:: 4.3 - .. deprecated:: 5.1 + .. versionchanged::: 6.0 - The ``callback`` argument is deprecated and will be removed in 6.0. - Use the returned awaitable object instead. + The ``callback`` argument was removed. Use the returned awaitable object instead. """ all_args = {} if access_token: @@ -710,25 +648,17 @@ class OAuth2Mixin(object): all_args.update(args) if all_args: - url += "?" + urllib_parse.urlencode(all_args) - callback = wrap(functools.partial(self._on_oauth2_request, callback)) + url += "?" + urllib.parse.urlencode(all_args) http = self.get_auth_http_client() if post_args is not None: - fut = http.fetch(url, method="POST", body=urllib_parse.urlencode(post_args)) + response = await http.fetch( + url, method="POST", body=urllib.parse.urlencode(post_args) + ) else: - fut = http.fetch(url) - fut.add_done_callback(callback) + response = await http.fetch(url) + return escape.json_decode(response.body) - def _on_oauth2_request(self, future, response_fut): - try: - response = response_fut.result() - except Exception as e: - future.set_exception(AuthError("Error response %s" % e)) - return - - future_set_result_unless_cancelled(future, escape.json_decode(response.body)) - - def get_auth_http_client(self): + def get_auth_http_client(self) -> httpclient.AsyncHTTPClient: """Returns the `.AsyncHTTPClient` instance to be used for auth requests. May be overridden by subclasses to use an HTTP client other than @@ -771,6 +701,7 @@ class TwitterMixin(OAuthMixin): and all of the custom Twitter user attributes described at https://dev.twitter.com/docs/api/1.1/get/users/show """ + _OAUTH_REQUEST_TOKEN_URL = "https://api.twitter.com/oauth/request_token" _OAUTH_ACCESS_TOKEN_URL = "https://api.twitter.com/oauth/access_token" _OAUTH_AUTHORIZE_URL = "https://api.twitter.com/oauth/authorize" @@ -778,8 +709,7 @@ class TwitterMixin(OAuthMixin): _OAUTH_NO_CALLBACKS = False _TWITTER_BASE_URL = "https://api.twitter.com/1.1" - @_non_deprecated_return_future - def authenticate_redirect(self, callback_uri=None, callback=None): + async def authenticate_redirect(self, callback_uri: str = None) -> None: """Just like `~OAuthMixin.authorize_redirect`, but auto-redirects if authorized. @@ -790,20 +720,24 @@ class TwitterMixin(OAuthMixin): Now returns a `.Future` and takes an optional callback, for compatibility with `.gen.coroutine`. - .. deprecated:: 5.1 + .. versionchanged:: 6.0 - The ``callback`` argument is deprecated and will be removed in 6.0. - Use the returned awaitable object instead. + The ``callback`` argument was removed. Use the returned + awaitable object instead. """ http = self.get_auth_http_client() - fut = http.fetch(self._oauth_request_token_url(callback_uri=callback_uri)) - fut.add_done_callback(wrap(functools.partial( - self._on_request_token, self._OAUTH_AUTHENTICATE_URL, - None, callback))) + response = await http.fetch( + self._oauth_request_token_url(callback_uri=callback_uri) + ) + self._on_request_token(self._OAUTH_AUTHENTICATE_URL, None, response) - @_auth_return_future - def twitter_request(self, path, callback=None, access_token=None, - post_args=None, **args): + async def twitter_request( + self, + path: str, + access_token: Dict[str, Any], + post_args: Dict[str, Any] = None, + **args: Any + ) -> Any: """Fetches the given API path, e.g., ``statuses/user_timeline/btaylor`` The path should not include the format or API version number. @@ -840,12 +774,12 @@ class TwitterMixin(OAuthMixin): .. testoutput:: :hide: - .. deprecated:: 5.1 + .. versionchanged:: 6.0 - The ``callback`` argument is deprecated and will be removed in 6.0. - Use the returned awaitable object instead. + The ``callback`` argument was removed. Use the returned + awaitable object instead. """ - if path.startswith('http:') or path.startswith('https:'): + if path.startswith("http:") or path.startswith("https:"): # Raw urls are useful for e.g. search which doesn't follow the # usual pattern: http://search.twitter.com/search.json url = path @@ -858,42 +792,38 @@ class TwitterMixin(OAuthMixin): all_args.update(post_args or {}) method = "POST" if post_args is not None else "GET" oauth = self._oauth_request_parameters( - url, access_token, all_args, method=method) + url, access_token, all_args, method=method + ) args.update(oauth) if args: - url += "?" + urllib_parse.urlencode(args) + url += "?" + urllib.parse.urlencode(args) http = self.get_auth_http_client() - http_callback = wrap(functools.partial(self._on_twitter_request, callback, url)) if post_args is not None: - fut = http.fetch(url, method="POST", body=urllib_parse.urlencode(post_args)) + response = await http.fetch( + url, method="POST", body=urllib.parse.urlencode(post_args) + ) else: - fut = http.fetch(url) - fut.add_done_callback(http_callback) + response = await http.fetch(url) + return escape.json_decode(response.body) - def _on_twitter_request(self, future, url, response_fut): - try: - response = response_fut.result() - except Exception as e: - future.set_exception(AuthError( - "Error response %s fetching %s" % (e, url))) - return - future_set_result_unless_cancelled(future, escape.json_decode(response.body)) - - def _oauth_consumer_token(self): - self.require_setting("twitter_consumer_key", "Twitter OAuth") - self.require_setting("twitter_consumer_secret", "Twitter OAuth") + def _oauth_consumer_token(self) -> Dict[str, Any]: + handler = cast(RequestHandler, self) + handler.require_setting("twitter_consumer_key", "Twitter OAuth") + handler.require_setting("twitter_consumer_secret", "Twitter OAuth") return dict( - key=self.settings["twitter_consumer_key"], - secret=self.settings["twitter_consumer_secret"]) + key=handler.settings["twitter_consumer_key"], + secret=handler.settings["twitter_consumer_secret"], + ) - @gen.coroutine - def _oauth_get_user_future(self, access_token): - user = yield self.twitter_request( - "/account/verify_credentials", - access_token=access_token) + async def _oauth_get_user_future( + self, access_token: Dict[str, Any] + ) -> Dict[str, Any]: + user = await self.twitter_request( + "/account/verify_credentials", access_token=access_token + ) if user: user["username"] = user["screen_name"] - raise gen.Return(user) + return user class GoogleOAuth2Mixin(OAuth2Mixin): @@ -910,18 +840,20 @@ class GoogleOAuth2Mixin(OAuth2Mixin): * In the OAuth section of the page, select Create New Client ID. * Set the Redirect URI to point to your auth handler * Copy the "Client secret" and "Client ID" to the application settings as - {"google_oauth": {"key": CLIENT_ID, "secret": CLIENT_SECRET}} + ``{"google_oauth": {"key": CLIENT_ID, "secret": CLIENT_SECRET}}`` .. versionadded:: 3.2 """ + _OAUTH_AUTHORIZE_URL = "https://accounts.google.com/o/oauth2/v2/auth" _OAUTH_ACCESS_TOKEN_URL = "https://www.googleapis.com/oauth2/v4/token" _OAUTH_USERINFO_URL = "https://www.googleapis.com/oauth2/v1/userinfo" _OAUTH_NO_CALLBACKS = False - _OAUTH_SETTINGS_KEY = 'google_oauth' + _OAUTH_SETTINGS_KEY = "google_oauth" - @_auth_return_future - def get_authenticated_user(self, redirect_uri, code, callback): + async def get_authenticated_user( + self, redirect_uri: str, code: str + ) -> Dict[str, Any]: """Handles the login for the Google user, returning an access token. The result is a dictionary containing an ``access_token`` field @@ -959,48 +891,47 @@ class GoogleOAuth2Mixin(OAuth2Mixin): .. testoutput:: :hide: - .. deprecated:: 5.1 + .. versionchanged:: 6.0 - The ``callback`` argument is deprecated and will be removed in 6.0. - Use the returned awaitable object instead. + The ``callback`` argument was removed. Use the returned awaitable object instead. """ # noqa: E501 + handler = cast(RequestHandler, self) http = self.get_auth_http_client() - body = urllib_parse.urlencode({ - "redirect_uri": redirect_uri, - "code": code, - "client_id": self.settings[self._OAUTH_SETTINGS_KEY]['key'], - "client_secret": self.settings[self._OAUTH_SETTINGS_KEY]['secret'], - "grant_type": "authorization_code", - }) + body = urllib.parse.urlencode( + { + "redirect_uri": redirect_uri, + "code": code, + "client_id": handler.settings[self._OAUTH_SETTINGS_KEY]["key"], + "client_secret": handler.settings[self._OAUTH_SETTINGS_KEY]["secret"], + "grant_type": "authorization_code", + } + ) - fut = http.fetch(self._OAUTH_ACCESS_TOKEN_URL, - method="POST", - headers={'Content-Type': 'application/x-www-form-urlencoded'}, - body=body) - fut.add_done_callback(wrap(functools.partial(self._on_access_token, callback))) - - def _on_access_token(self, future, response_fut): - """Callback function for the exchange to the access token.""" - try: - response = response_fut.result() - except Exception as e: - future.set_exception(AuthError('Google auth error: %s' % str(e))) - return - - args = escape.json_decode(response.body) - future_set_result_unless_cancelled(future, args) + response = await http.fetch( + self._OAUTH_ACCESS_TOKEN_URL, + method="POST", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + body=body, + ) + return escape.json_decode(response.body) class FacebookGraphMixin(OAuth2Mixin): """Facebook authentication using the new Graph API and OAuth2.""" + _OAUTH_ACCESS_TOKEN_URL = "https://graph.facebook.com/oauth/access_token?" _OAUTH_AUTHORIZE_URL = "https://www.facebook.com/dialog/oauth?" _OAUTH_NO_CALLBACKS = False _FACEBOOK_BASE_URL = "https://graph.facebook.com" - @_auth_return_future - def get_authenticated_user(self, redirect_uri, client_id, client_secret, - code, callback, extra_fields=None): + async def get_authenticated_user( + self, + redirect_uri: str, + client_id: str, + client_secret: str, + code: str, + extra_fields: Dict[str, Any] = None, + ) -> Optional[Dict[str, Any]]: """Handles the login for the Facebook user, returning a user object. Example usage: @@ -1042,10 +973,9 @@ class FacebookGraphMixin(OAuth2Mixin): The ``session_expires`` field was updated to support changes made to the Facebook API in March 2017. - .. deprecated:: 5.1 + .. versionchanged:: 6.0 - The ``callback`` argument is deprecated and will be removed in 6.0. - Use the returned awaitable object instead. + The ``callback`` argument was removed. Use the returned awaitable object instead. """ http = self.get_auth_http_client() args = { @@ -1055,42 +985,35 @@ class FacebookGraphMixin(OAuth2Mixin): "client_secret": client_secret, } - fields = set(['id', 'name', 'first_name', 'last_name', - 'locale', 'picture', 'link']) + fields = set( + ["id", "name", "first_name", "last_name", "locale", "picture", "link"] + ) if extra_fields: fields.update(extra_fields) - fut = http.fetch(self._oauth_request_token_url(**args)) - fut.add_done_callback(wrap(functools.partial(self._on_access_token, redirect_uri, client_id, - client_secret, callback, fields))) - - @gen.coroutine - def _on_access_token(self, redirect_uri, client_id, client_secret, - future, fields, response_fut): - try: - response = response_fut.result() - except Exception as e: - future.set_exception(AuthError('Facebook auth error: %s' % str(e))) - return - + response = await http.fetch( + self._oauth_request_token_url(**args) # type: ignore + ) args = escape.json_decode(response.body) session = { "access_token": args.get("access_token"), - "expires_in": args.get("expires_in") + "expires_in": args.get("expires_in"), } + assert session["access_token"] is not None - user = yield self.facebook_request( + user = await self.facebook_request( path="/me", access_token=session["access_token"], - appsecret_proof=hmac.new(key=client_secret.encode('utf8'), - msg=session["access_token"].encode('utf8'), - digestmod=hashlib.sha256).hexdigest(), - fields=",".join(fields) + appsecret_proof=hmac.new( + key=client_secret.encode("utf8"), + msg=session["access_token"].encode("utf8"), + digestmod=hashlib.sha256, + ).hexdigest(), + fields=",".join(fields), ) if user is None: - future_set_result_unless_cancelled(future, None) - return + return None fieldmap = {} for field in fields: @@ -1100,13 +1023,21 @@ class FacebookGraphMixin(OAuth2Mixin): # older versions in which the server used url-encoding and # this code simply returned the string verbatim. # This should change in Tornado 5.0. - fieldmap.update({"access_token": session["access_token"], - "session_expires": str(session.get("expires_in"))}) - future_set_result_unless_cancelled(future, fieldmap) + fieldmap.update( + { + "access_token": session["access_token"], + "session_expires": str(session.get("expires_in")), + } + ) + return fieldmap - @_auth_return_future - def facebook_request(self, path, callback, access_token=None, - post_args=None, **args): + async def facebook_request( + self, + path: str, + access_token: str = None, + post_args: Dict[str, Any] = None, + **args: Any + ) -> Any: """Fetches the given relative API path, e.g., "/btaylor/picture" If the request is a POST, ``post_args`` should be provided. Query @@ -1153,35 +1084,39 @@ class FacebookGraphMixin(OAuth2Mixin): .. versionchanged:: 3.1 Added the ability to override ``self._FACEBOOK_BASE_URL``. - .. deprecated:: 5.1 + .. versionchanged:: 6.0 - The ``callback`` argument is deprecated and will be removed in 6.0. - Use the returned awaitable object instead. + The ``callback`` argument was removed. Use the returned awaitable object instead. """ url = self._FACEBOOK_BASE_URL + path - # Thanks to the _auth_return_future decorator, our "callback" - # argument is a Future, which we cannot pass as a callback to - # oauth2_request. Instead, have oauth2_request return a - # future and chain them together. - oauth_future = self.oauth2_request(url, access_token=access_token, - post_args=post_args, **args) - chain_future(oauth_future, callback) + return await self.oauth2_request( + url, access_token=access_token, post_args=post_args, **args + ) -def _oauth_signature(consumer_token, method, url, parameters={}, token=None): +def _oauth_signature( + consumer_token: Dict[str, Any], + method: str, + url: str, + parameters: Dict[str, Any] = {}, + token: Dict[str, Any] = None, +) -> bytes: """Calculates the HMAC-SHA1 OAuth signature for the given request. See http://oauth.net/core/1.0/#signing_process """ - parts = urlparse.urlparse(url) + parts = urllib.parse.urlparse(url) scheme, netloc, path = parts[:3] normalized_url = scheme.lower() + "://" + netloc.lower() + path base_elems = [] base_elems.append(method.upper()) base_elems.append(normalized_url) - base_elems.append("&".join("%s=%s" % (k, _oauth_escape(str(v))) - for k, v in sorted(parameters.items()))) + base_elems.append( + "&".join( + "%s=%s" % (k, _oauth_escape(str(v))) for k, v in sorted(parameters.items()) + ) + ) base_string = "&".join(_oauth_escape(e) for e in base_elems) key_elems = [escape.utf8(consumer_token["secret"])] @@ -1192,42 +1127,53 @@ def _oauth_signature(consumer_token, method, url, parameters={}, token=None): return binascii.b2a_base64(hash.digest())[:-1] -def _oauth10a_signature(consumer_token, method, url, parameters={}, token=None): +def _oauth10a_signature( + consumer_token: Dict[str, Any], + method: str, + url: str, + parameters: Dict[str, Any] = {}, + token: Dict[str, Any] = None, +) -> bytes: """Calculates the HMAC-SHA1 OAuth 1.0a signature for the given request. See http://oauth.net/core/1.0a/#signing_process """ - parts = urlparse.urlparse(url) + parts = urllib.parse.urlparse(url) scheme, netloc, path = parts[:3] normalized_url = scheme.lower() + "://" + netloc.lower() + path base_elems = [] base_elems.append(method.upper()) base_elems.append(normalized_url) - base_elems.append("&".join("%s=%s" % (k, _oauth_escape(str(v))) - for k, v in sorted(parameters.items()))) + base_elems.append( + "&".join( + "%s=%s" % (k, _oauth_escape(str(v))) for k, v in sorted(parameters.items()) + ) + ) base_string = "&".join(_oauth_escape(e) for e in base_elems) - key_elems = [escape.utf8(urllib_parse.quote(consumer_token["secret"], safe='~'))] - key_elems.append(escape.utf8(urllib_parse.quote(token["secret"], safe='~') if token else "")) + key_elems = [escape.utf8(urllib.parse.quote(consumer_token["secret"], safe="~"))] + key_elems.append( + escape.utf8(urllib.parse.quote(token["secret"], safe="~") if token else "") + ) key = b"&".join(key_elems) hash = hmac.new(key, escape.utf8(base_string), hashlib.sha1) return binascii.b2a_base64(hash.digest())[:-1] -def _oauth_escape(val): +def _oauth_escape(val: Union[str, bytes]) -> str: if isinstance(val, unicode_type): val = val.encode("utf-8") - return urllib_parse.quote(val, safe="~") + return urllib.parse.quote(val, safe="~") -def _oauth_parse_response(body): +def _oauth_parse_response(body: bytes) -> Dict[str, Any]: # I can't find an officially-defined encoding for oauth responses and # have never seen anyone use non-ascii. Leave the response in a byte # string for python 2, and use utf8 on python 3. - body = escape.native_str(body) - p = urlparse.parse_qs(body, keep_blank_values=False) + body_str = escape.native_str(body) + p = urllib.parse.parse_qs(body_str, keep_blank_values=False) token = dict(key=p["oauth_token"][0], secret=p["oauth_token_secret"][0]) # Add the extra parameters the Provider included to the token diff --git a/server/www/packages/packages-linux/x64/tornado/autoreload.py b/server/www/packages/packages-linux/x64/tornado/autoreload.py index 7d69474..1c47aae 100644 --- a/server/www/packages/packages-linux/x64/tornado/autoreload.py +++ b/server/www/packages/packages-linux/x64/tornado/autoreload.py @@ -33,9 +33,8 @@ This combination is encouraged as the wrapper catches syntax errors and other import-time failures, while debug mode catches changes once the server has started. -This module depends on `.IOLoop`, so it will not work in WSGI applications -and Google App Engine. It also will not work correctly when `.HTTPServer`'s -multi-process mode is used. +This module will not work correctly when `.HTTPServer`'s multi-process +mode is used. Reloading loses any Python interpreter command-line arguments (e.g. ``-u``) because it re-executes Python using ``sys.executable`` and ``sys.argv``. @@ -44,8 +43,6 @@ incorrectly. """ -from __future__ import absolute_import, division, print_function - import os import sys @@ -96,23 +93,29 @@ from tornado.util import exec_in try: import signal except ImportError: - signal = None + signal = None # type: ignore + +import typing +from typing import Callable, Dict + +if typing.TYPE_CHECKING: + from typing import List, Optional, Union # noqa: F401 # os.execv is broken on Windows and can't properly parse command line # arguments and executable name if they contain whitespaces. subprocess # fixes that behavior. -_has_execv = sys.platform != 'win32' +_has_execv = sys.platform != "win32" _watched_files = set() _reload_hooks = [] _reload_attempted = False _io_loops = weakref.WeakKeyDictionary() # type: ignore _autoreload_is_main = False -_original_argv = None +_original_argv = None # type: Optional[List[str]] _original_spec = None -def start(check_time=500): +def start(check_time: int = 500) -> None: """Begins watching source files for changes. .. versionchanged:: 5.0 @@ -124,13 +127,13 @@ def start(check_time=500): _io_loops[io_loop] = True if len(_io_loops) > 1: gen_log.warning("tornado.autoreload started more than once in the same process") - modify_times = {} + modify_times = {} # type: Dict[str, float] callback = functools.partial(_reload_on_update, modify_times) scheduler = ioloop.PeriodicCallback(callback, check_time) scheduler.start() -def wait(): +def wait() -> None: """Wait for a watched file to change, then restart the process. Intended to be used at the end of scripts like unit test runners, @@ -142,7 +145,7 @@ def wait(): io_loop.start() -def watch(filename): +def watch(filename: str) -> None: """Add a file to the watch list. All imported modules are watched by default. @@ -150,7 +153,7 @@ def watch(filename): _watched_files.add(filename) -def add_reload_hook(fn): +def add_reload_hook(fn: Callable[[], None]) -> None: """Add a function to be called before reloading the process. Note that for open file and socket handles it is generally @@ -161,7 +164,7 @@ def add_reload_hook(fn): _reload_hooks.append(fn) -def _reload_on_update(modify_times): +def _reload_on_update(modify_times: Dict[str, float]) -> None: if _reload_attempted: # We already tried to reload and it didn't work, so don't try again. return @@ -187,7 +190,7 @@ def _reload_on_update(modify_times): _check_file(modify_times, path) -def _check_file(modify_times, path): +def _check_file(modify_times: Dict[str, float], path: str) -> None: try: modified = os.stat(path).st_mtime except Exception: @@ -200,7 +203,7 @@ def _check_file(modify_times, path): _reload() -def _reload(): +def _reload() -> None: global _reload_attempted _reload_attempted = True for fn in _reload_hooks: @@ -218,19 +221,20 @@ def _reload(): # sys.path[0] is an empty string and add the current directory to # $PYTHONPATH. if _autoreload_is_main: + assert _original_argv is not None spec = _original_spec argv = _original_argv else: - spec = getattr(sys.modules['__main__'], '__spec__', None) + spec = getattr(sys.modules["__main__"], "__spec__", None) argv = sys.argv if spec: - argv = ['-m', spec.name] + argv[1:] + argv = ["-m", spec.name] + argv[1:] else: - path_prefix = '.' + os.pathsep - if (sys.path[0] == '' and - not os.environ.get("PYTHONPATH", "").startswith(path_prefix)): - os.environ["PYTHONPATH"] = (path_prefix + - os.environ.get("PYTHONPATH", "")) + path_prefix = "." + os.pathsep + if sys.path[0] == "" and not os.environ.get("PYTHONPATH", "").startswith( + path_prefix + ): + os.environ["PYTHONPATH"] = path_prefix + os.environ.get("PYTHONPATH", "") if not _has_execv: subprocess.Popen([sys.executable] + argv) os._exit(0) @@ -249,7 +253,9 @@ def _reload(): # Unfortunately the errno returned in this case does not # appear to be consistent, so we can't easily check for # this error specifically. - os.spawnv(os.P_NOWAIT, sys.executable, [sys.executable] + argv) + os.spawnv( # type: ignore + os.P_NOWAIT, sys.executable, [sys.executable] + argv + ) # At this point the IOLoop has been closed and finally # blocks will experience errors if we allow the stack to # unwind, so just exit uncleanly. @@ -263,7 +269,7 @@ Usage: """ -def main(): +def main() -> None: """Command-line wrapper to re-run a script whenever its source changes. Scripts may be specified by filename or module name:: @@ -280,12 +286,13 @@ def main(): # The main module can be tricky; set the variables both in our globals # (which may be __main__) and the real importable version. import tornado.autoreload + global _autoreload_is_main global _original_argv, _original_spec tornado.autoreload._autoreload_is_main = _autoreload_is_main = True original_argv = sys.argv tornado.autoreload._original_argv = _original_argv = original_argv - original_spec = getattr(sys.modules['__main__'], '__spec__', None) + original_spec = getattr(sys.modules["__main__"], "__spec__", None) tornado.autoreload._original_spec = _original_spec = original_spec sys.argv = sys.argv[:] if len(sys.argv) >= 3 and sys.argv[1] == "-m": @@ -303,6 +310,7 @@ def main(): try: if mode == "module": import runpy + runpy.run_module(module, run_name="__main__", alter_sys=True) elif mode == "script": with open(script) as f: @@ -340,12 +348,12 @@ def main(): # restore sys.argv so subsequent executions will include autoreload sys.argv = original_argv - if mode == 'module': + if mode == "module": # runpy did a fake import of the module as __main__, but now it's # no longer in sys.modules. Figure out where it is and watch it. loader = pkgutil.get_loader(module) if loader is not None: - watch(loader.get_filename()) + watch(loader.get_filename()) # type: ignore wait() diff --git a/server/www/packages/packages-linux/x64/tornado/concurrent.py b/server/www/packages/packages-linux/x64/tornado/concurrent.py index f7e6bcc..3a49940 100644 --- a/server/www/packages/packages-linux/x64/tornado/concurrent.py +++ b/server/www/packages/packages-linux/x64/tornado/concurrent.py @@ -14,389 +14,64 @@ # under the License. """Utilities for working with ``Future`` objects. -``Futures`` are a pattern for concurrent programming introduced in -Python 3.2 in the `concurrent.futures` package, and also adopted (in a -slightly different form) in Python 3.4's `asyncio` package. This -package defines a ``Future`` class that is an alias for `asyncio.Future` -when available, and a compatible implementation for older versions of -Python. It also includes some utility functions for interacting with -``Future`` objects. +Tornado previously provided its own ``Future`` class, but now uses +`asyncio.Future`. This module contains utility functions for working +with `asyncio.Future` in a way that is backwards-compatible with +Tornado's old ``Future`` implementation. -While this package is an important part of Tornado's internal +While this module is an important part of Tornado's internal implementation, applications rarely need to interact with it directly. -""" -from __future__ import absolute_import, division, print_function +""" + +import asyncio +from concurrent import futures import functools -import platform -import textwrap -import traceback import sys -import warnings +import types from tornado.log import app_log -from tornado.stack_context import ExceptionStackContext, wrap -from tornado.util import raise_exc_info, ArgReplacer, is_finalizing -try: - from concurrent import futures -except ImportError: - futures = None +import typing +from typing import Any, Callable, Optional, Tuple, Union -try: - import asyncio -except ImportError: - asyncio = None - -try: - import typing -except ImportError: - typing = None - - -# Can the garbage collector handle cycles that include __del__ methods? -# This is true in cpython beginning with version 3.4 (PEP 442). -_GC_CYCLE_FINALIZERS = (platform.python_implementation() == 'CPython' and - sys.version_info >= (3, 4)) +_T = typing.TypeVar("_T") class ReturnValueIgnoredError(Exception): + # No longer used; was previously used by @return_future pass -# This class and associated code in the future object is derived -# from the Trollius project, a backport of asyncio to Python 2.x - 3.x + +Future = asyncio.Future + +FUTURES = (futures.Future, Future) -class _TracebackLogger(object): - """Helper to log a traceback upon destruction if not cleared. - - This solves a nasty problem with Futures and Tasks that have an - exception set: if nobody asks for the exception, the exception is - never logged. This violates the Zen of Python: 'Errors should - never pass silently. Unless explicitly silenced.' - - However, we don't want to log the exception as soon as - set_exception() is called: if the calling code is written - properly, it will get the exception and handle it properly. But - we *do* want to log it if result() or exception() was never called - -- otherwise developers waste a lot of time wondering why their - buggy code fails silently. - - An earlier attempt added a __del__() method to the Future class - itself, but this backfired because the presence of __del__() - prevents garbage collection from breaking cycles. A way out of - this catch-22 is to avoid having a __del__() method on the Future - class itself, but instead to have a reference to a helper object - with a __del__() method that logs the traceback, where we ensure - that the helper object doesn't participate in cycles, and only the - Future has a reference to it. - - The helper object is added when set_exception() is called. When - the Future is collected, and the helper is present, the helper - object is also collected, and its __del__() method will log the - traceback. When the Future's result() or exception() method is - called (and a helper object is present), it removes the the helper - object, after calling its clear() method to prevent it from - logging. - - One downside is that we do a fair amount of work to extract the - traceback from the exception, even when it is never logged. It - would seem cheaper to just store the exception object, but that - references the traceback, which references stack frames, which may - reference the Future, which references the _TracebackLogger, and - then the _TracebackLogger would be included in a cycle, which is - what we're trying to avoid! As an optimization, we don't - immediately format the exception; we only do the work when - activate() is called, which call is delayed until after all the - Future's callbacks have run. Since usually a Future has at least - one callback (typically set by 'yield From') and usually that - callback extracts the callback, thereby removing the need to - format the exception. - - PS. I don't claim credit for this solution. I first heard of it - in a discussion about closing files when they are collected. - """ - - __slots__ = ('exc_info', 'formatted_tb') - - def __init__(self, exc_info): - self.exc_info = exc_info - self.formatted_tb = None - - def activate(self): - exc_info = self.exc_info - if exc_info is not None: - self.exc_info = None - self.formatted_tb = traceback.format_exception(*exc_info) - - def clear(self): - self.exc_info = None - self.formatted_tb = None - - def __del__(self, is_finalizing=is_finalizing): - if not is_finalizing() and self.formatted_tb: - app_log.error('Future exception was never retrieved: %s', - ''.join(self.formatted_tb).rstrip()) - - -class Future(object): - """Placeholder for an asynchronous result. - - A ``Future`` encapsulates the result of an asynchronous - operation. In synchronous applications ``Futures`` are used - to wait for the result from a thread or process pool; in - Tornado they are normally used with `.IOLoop.add_future` or by - yielding them in a `.gen.coroutine`. - - `tornado.concurrent.Future` is an alias for `asyncio.Future` when - that package is available (Python 3.4+). Unlike - `concurrent.futures.Future`, the ``Futures`` used by Tornado and - `asyncio` are not thread-safe (and therefore faster for use with - single-threaded event loops). - - In addition to ``exception`` and ``set_exception``, Tornado's - ``Future`` implementation supports storing an ``exc_info`` triple - to support better tracebacks on Python 2. To set an ``exc_info`` - triple, use `future_set_exc_info`, and to retrieve one, call - `result()` (which will raise it). - - .. versionchanged:: 4.0 - `tornado.concurrent.Future` is always a thread-unsafe ``Future`` - with support for the ``exc_info`` methods. Previously it would - be an alias for the thread-safe `concurrent.futures.Future` - if that package was available and fall back to the thread-unsafe - implementation if it was not. - - .. versionchanged:: 4.1 - If a `.Future` contains an error but that error is never observed - (by calling ``result()``, ``exception()``, or ``exc_info()``), - a stack trace will be logged when the `.Future` is garbage collected. - This normally indicates an error in the application, but in cases - where it results in undesired logging it may be necessary to - suppress the logging by ensuring that the exception is observed: - ``f.add_done_callback(lambda f: f.exception())``. - - .. versionchanged:: 5.0 - - This class was previoiusly available under the name - ``TracebackFuture``. This name, which was deprecated since - version 4.0, has been removed. When `asyncio` is available - ``tornado.concurrent.Future`` is now an alias for - `asyncio.Future`. Like `asyncio.Future`, callbacks are now - always scheduled on the `.IOLoop` and are never run - synchronously. - - """ - def __init__(self): - self._done = False - self._result = None - self._exc_info = None - - self._log_traceback = False # Used for Python >= 3.4 - self._tb_logger = None # Used for Python <= 3.3 - - self._callbacks = [] - - # Implement the Python 3.5 Awaitable protocol if possible - # (we can't use return and yield together until py33). - if sys.version_info >= (3, 3): - exec(textwrap.dedent(""" - def __await__(self): - return (yield self) - """)) - else: - # Py2-compatible version for use with cython. - def __await__(self): - result = yield self - # StopIteration doesn't take args before py33, - # but Cython recognizes the args tuple. - e = StopIteration() - e.args = (result,) - raise e - - def cancel(self): - """Cancel the operation, if possible. - - Tornado ``Futures`` do not support cancellation, so this method always - returns False. - """ - return False - - def cancelled(self): - """Returns True if the operation has been cancelled. - - Tornado ``Futures`` do not support cancellation, so this method - always returns False. - """ - return False - - def running(self): - """Returns True if this operation is currently running.""" - return not self._done - - def done(self): - """Returns True if the future has finished running.""" - return self._done - - def _clear_tb_log(self): - self._log_traceback = False - if self._tb_logger is not None: - self._tb_logger.clear() - self._tb_logger = None - - def result(self, timeout=None): - """If the operation succeeded, return its result. If it failed, - re-raise its exception. - - This method takes a ``timeout`` argument for compatibility with - `concurrent.futures.Future` but it is an error to call it - before the `Future` is done, so the ``timeout`` is never used. - """ - self._clear_tb_log() - if self._result is not None: - return self._result - if self._exc_info is not None: - try: - raise_exc_info(self._exc_info) - finally: - self = None - self._check_done() - return self._result - - def exception(self, timeout=None): - """If the operation raised an exception, return the `Exception` - object. Otherwise returns None. - - This method takes a ``timeout`` argument for compatibility with - `concurrent.futures.Future` but it is an error to call it - before the `Future` is done, so the ``timeout`` is never used. - """ - self._clear_tb_log() - if self._exc_info is not None: - return self._exc_info[1] - else: - self._check_done() - return None - - def add_done_callback(self, fn): - """Attaches the given callback to the `Future`. - - It will be invoked with the `Future` as its argument when the Future - has finished running and its result is available. In Tornado - consider using `.IOLoop.add_future` instead of calling - `add_done_callback` directly. - """ - if self._done: - from tornado.ioloop import IOLoop - IOLoop.current().add_callback(fn, self) - else: - self._callbacks.append(fn) - - def set_result(self, result): - """Sets the result of a ``Future``. - - It is undefined to call any of the ``set`` methods more than once - on the same object. - """ - self._result = result - self._set_done() - - def set_exception(self, exception): - """Sets the exception of a ``Future.``""" - self.set_exc_info( - (exception.__class__, - exception, - getattr(exception, '__traceback__', None))) - - def exc_info(self): - """Returns a tuple in the same format as `sys.exc_info` or None. - - .. versionadded:: 4.0 - """ - self._clear_tb_log() - return self._exc_info - - def set_exc_info(self, exc_info): - """Sets the exception information of a ``Future.`` - - Preserves tracebacks on Python 2. - - .. versionadded:: 4.0 - """ - self._exc_info = exc_info - self._log_traceback = True - if not _GC_CYCLE_FINALIZERS: - self._tb_logger = _TracebackLogger(exc_info) - - try: - self._set_done() - finally: - # Activate the logger after all callbacks have had a - # chance to call result() or exception(). - if self._log_traceback and self._tb_logger is not None: - self._tb_logger.activate() - self._exc_info = exc_info - - def _check_done(self): - if not self._done: - raise Exception("DummyFuture does not support blocking for results") - - def _set_done(self): - self._done = True - if self._callbacks: - from tornado.ioloop import IOLoop - loop = IOLoop.current() - for cb in self._callbacks: - loop.add_callback(cb, self) - self._callbacks = None - - # On Python 3.3 or older, objects with a destructor part of a reference - # cycle are never destroyed. It's no longer the case on Python 3.4 thanks to - # the PEP 442. - if _GC_CYCLE_FINALIZERS: - def __del__(self, is_finalizing=is_finalizing): - if is_finalizing() or not self._log_traceback: - # set_exception() was not called, or result() or exception() - # has consumed the exception - return - - tb = traceback.format_exception(*self._exc_info) - - app_log.error('Future %r exception was never retrieved: %s', - self, ''.join(tb).rstrip()) - - -if asyncio is not None: - Future = asyncio.Future # noqa - -if futures is None: - FUTURES = Future # type: typing.Union[type, typing.Tuple[type, ...]] -else: - FUTURES = (futures.Future, Future) - - -def is_future(x): +def is_future(x: Any) -> bool: return isinstance(x, FUTURES) -class DummyExecutor(object): - def submit(self, fn, *args, **kwargs): - future = Future() +class DummyExecutor(futures.Executor): + def submit( + self, fn: Callable[..., _T], *args: Any, **kwargs: Any + ) -> "futures.Future[_T]": + future = futures.Future() # type: futures.Future[_T] try: future_set_result_unless_cancelled(future, fn(*args, **kwargs)) except Exception: future_set_exc_info(future, sys.exc_info()) return future - def shutdown(self, wait=True): + def shutdown(self, wait: bool = True) -> None: pass dummy_executor = DummyExecutor() -def run_on_executor(*args, **kwargs): +def run_on_executor(*args: Any, **kwargs: Any) -> Callable: """Decorator to run a synchronous method asynchronously on an executor. The decorated method may be called with a ``callback`` keyword @@ -432,24 +107,25 @@ def run_on_executor(*args, **kwargs): The ``callback`` argument is deprecated and will be removed in 6.0. The decorator itself is discouraged in new code but will not be removed in 6.0. + + .. versionchanged:: 6.0 + + The ``callback`` argument was removed. """ - def run_on_executor_decorator(fn): + # Fully type-checking decorators is tricky, and this one is + # discouraged anyway so it doesn't have all the generic magic. + def run_on_executor_decorator(fn: Callable) -> Callable[..., Future]: executor = kwargs.get("executor", "executor") @functools.wraps(fn) - def wrapper(self, *args, **kwargs): - callback = kwargs.pop("callback", None) - async_future = Future() + def wrapper(self: Any, *args: Any, **kwargs: Any) -> Future: + async_future = Future() # type: Future conc_future = getattr(self, executor).submit(fn, self, *args, **kwargs) chain_future(conc_future, async_future) - if callback: - warnings.warn("callback arguments are deprecated, use the returned Future instead", - DeprecationWarning) - from tornado.ioloop import IOLoop - IOLoop.current().add_future( - async_future, lambda future: callback(future.result())) return async_future + return wrapper + if args and kwargs: raise ValueError("cannot combine positional and keyword args") if len(args) == 1: @@ -462,129 +138,7 @@ def run_on_executor(*args, **kwargs): _NO_RESULT = object() -def return_future(f): - """Decorator to make a function that returns via callback return a - `Future`. - - This decorator was provided to ease the transition from - callback-oriented code to coroutines. It is not recommended for - new code. - - The wrapped function should take a ``callback`` keyword argument - and invoke it with one argument when it has finished. To signal failure, - the function can simply raise an exception (which will be - captured by the `.StackContext` and passed along to the ``Future``). - - From the caller's perspective, the callback argument is optional. - If one is given, it will be invoked when the function is complete - with ``Future.result()`` as an argument. If the function fails, the - callback will not be run and an exception will be raised into the - surrounding `.StackContext`. - - If no callback is given, the caller should use the ``Future`` to - wait for the function to complete (perhaps by yielding it in a - coroutine, or passing it to `.IOLoop.add_future`). - - Usage: - - .. testcode:: - - @return_future - def future_func(arg1, arg2, callback): - # Do stuff (possibly asynchronous) - callback(result) - - async def caller(): - await future_func(arg1, arg2) - - .. - - Note that ``@return_future`` and ``@gen.engine`` can be applied to the - same function, provided ``@return_future`` appears first. However, - consider using ``@gen.coroutine`` instead of this combination. - - .. versionchanged:: 5.1 - - Now raises a `.DeprecationWarning` if a callback argument is passed to - the decorated function and deprecation warnings are enabled. - - .. deprecated:: 5.1 - - This decorator will be removed in Tornado 6.0. New code should - use coroutines directly instead of wrapping callback-based code - with this decorator. Interactions with non-Tornado - callback-based code should be managed explicitly to avoid - relying on the `.ExceptionStackContext` built into this - decorator. - """ - warnings.warn("@return_future is deprecated, use coroutines instead", - DeprecationWarning) - return _non_deprecated_return_future(f, warn=True) - - -def _non_deprecated_return_future(f, warn=False): - # Allow auth.py to use this decorator without triggering - # deprecation warnings. This will go away once auth.py has removed - # its legacy interfaces in 6.0. - replacer = ArgReplacer(f, 'callback') - - @functools.wraps(f) - def wrapper(*args, **kwargs): - future = Future() - callback, args, kwargs = replacer.replace( - lambda value=_NO_RESULT: future_set_result_unless_cancelled(future, value), - args, kwargs) - - def handle_error(typ, value, tb): - future_set_exc_info(future, (typ, value, tb)) - return True - exc_info = None - esc = ExceptionStackContext(handle_error, delay_warning=True) - with esc: - if not warn: - # HACK: In non-deprecated mode (only used in auth.py), - # suppress the warning entirely. Since this is added - # in a 5.1 patch release and already removed in 6.0 - # I'm prioritizing a minimial change instead of a - # clean solution. - esc.delay_warning = False - try: - result = f(*args, **kwargs) - if result is not None: - raise ReturnValueIgnoredError( - "@return_future should not be used with functions " - "that return values") - except: - exc_info = sys.exc_info() - raise - if exc_info is not None: - # If the initial synchronous part of f() raised an exception, - # go ahead and raise it to the caller directly without waiting - # for them to inspect the Future. - future.result() - - # If the caller passed in a callback, schedule it to be called - # when the future resolves. It is important that this happens - # just before we return the future, or else we risk confusing - # stack contexts with multiple exceptions (one here with the - # immediate exception, and again when the future resolves and - # the callback triggers its exception by calling future.result()). - if callback is not None: - warnings.warn("callback arguments are deprecated, use the returned Future instead", - DeprecationWarning) - - def run_callback(future): - result = future.result() - if result is _NO_RESULT: - callback() - else: - callback(future.result()) - future_add_done_callback(future, wrap(run_callback)) - return future - return wrapper - - -def chain_future(a, b): +def chain_future(a: "Future[_T]", b: "Future[_T]") -> None: """Chain two futures together so that when one completes, so does the other. The result (success or failure) of ``a`` will be copied to ``b``, unless @@ -596,29 +150,33 @@ def chain_future(a, b): `concurrent.futures.Future`. """ - def copy(future): + + def copy(future: "Future[_T]") -> None: assert future is a if b.done(): return - if (hasattr(a, 'exc_info') and - a.exc_info() is not None): - future_set_exc_info(b, a.exc_info()) + if hasattr(a, "exc_info") and a.exc_info() is not None: # type: ignore + future_set_exc_info(b, a.exc_info()) # type: ignore elif a.exception() is not None: b.set_exception(a.exception()) else: b.set_result(a.result()) + if isinstance(a, Future): future_add_done_callback(a, copy) else: # concurrent.futures.Future from tornado.ioloop import IOLoop + IOLoop.current().add_future(a, copy) -def future_set_result_unless_cancelled(future, value): +def future_set_result_unless_cancelled( + future: "Union[futures.Future[_T], Future[_T]]", value: _T +) -> None: """Set the given ``value`` as the `Future`'s result, if not cancelled. - Avoids asyncio.InvalidStateError when calling set_result() on + Avoids ``asyncio.InvalidStateError`` when calling ``set_result()`` on a cancelled `asyncio.Future`. .. versionadded:: 5.0 @@ -627,23 +185,69 @@ def future_set_result_unless_cancelled(future, value): future.set_result(value) -def future_set_exc_info(future, exc_info): +def future_set_exception_unless_cancelled( + future: "Union[futures.Future[_T], Future[_T]]", exc: BaseException +) -> None: + """Set the given ``exc`` as the `Future`'s exception. + + If the Future is already canceled, logs the exception instead. If + this logging is not desired, the caller should explicitly check + the state of the Future and call ``Future.set_exception`` instead of + this wrapper. + + Avoids ``asyncio.InvalidStateError`` when calling ``set_exception()`` on + a cancelled `asyncio.Future`. + + .. versionadded:: 6.0 + + """ + if not future.cancelled(): + future.set_exception(exc) + else: + app_log.error("Exception after Future was cancelled", exc_info=exc) + + +def future_set_exc_info( + future: "Union[futures.Future[_T], Future[_T]]", + exc_info: Tuple[ + Optional[type], Optional[BaseException], Optional[types.TracebackType] + ], +) -> None: """Set the given ``exc_info`` as the `Future`'s exception. - Understands both `asyncio.Future` and Tornado's extensions to - enable better tracebacks on Python 2. + Understands both `asyncio.Future` and the extensions in older + versions of Tornado to enable better tracebacks on Python 2. .. versionadded:: 5.0 + + .. versionchanged:: 6.0 + + If the future is already cancelled, this function is a no-op. + (previously ``asyncio.InvalidStateError`` would be raised) + """ - if hasattr(future, 'set_exc_info'): - # Tornado's Future - future.set_exc_info(exc_info) - else: - # asyncio.Future - future.set_exception(exc_info[1]) + if exc_info[1] is None: + raise Exception("future_set_exc_info called with no exception") + future_set_exception_unless_cancelled(future, exc_info[1]) -def future_add_done_callback(future, callback): +@typing.overload +def future_add_done_callback( + future: "futures.Future[_T]", callback: Callable[["futures.Future[_T]"], None] +) -> None: + pass + + +@typing.overload # noqa: F811 +def future_add_done_callback( + future: "Future[_T]", callback: Callable[["Future[_T]"], None] +) -> None: + pass + + +def future_add_done_callback( # noqa: F811 + future: "Union[futures.Future[_T], Future[_T]]", callback: Callable[..., None] +) -> None: """Arrange to call ``callback`` when ``future`` is complete. ``callback`` is invoked with one argument, the ``future``. diff --git a/server/www/packages/packages-linux/x64/tornado/curl_httpclient.py b/server/www/packages/packages-linux/x64/tornado/curl_httpclient.py index 7f5cb10..4119585 100644 --- a/server/www/packages/packages-linux/x64/tornado/curl_httpclient.py +++ b/server/www/packages/packages-linux/x64/tornado/curl_httpclient.py @@ -15,8 +15,6 @@ """Non-blocking HTTP client implementation using pycurl.""" -from __future__ import absolute_import, division, print_function - import collections import functools import logging @@ -27,32 +25,49 @@ from io import BytesIO from tornado import httputil from tornado import ioloop -from tornado import stack_context from tornado.escape import utf8, native_str -from tornado.httpclient import HTTPResponse, HTTPError, AsyncHTTPClient, main +from tornado.httpclient import ( + HTTPRequest, + HTTPResponse, + HTTPError, + AsyncHTTPClient, + main, +) +from tornado.log import app_log -curl_log = logging.getLogger('tornado.curl_httpclient') +from typing import Dict, Any, Callable, Union +import typing + +if typing.TYPE_CHECKING: + from typing import Deque, Tuple, Optional # noqa: F401 + +curl_log = logging.getLogger("tornado.curl_httpclient") class CurlAsyncHTTPClient(AsyncHTTPClient): - def initialize(self, max_clients=10, defaults=None): + def initialize( # type: ignore + self, max_clients: int = 10, defaults: Dict[str, Any] = None + ) -> None: super(CurlAsyncHTTPClient, self).initialize(defaults=defaults) self._multi = pycurl.CurlMulti() self._multi.setopt(pycurl.M_TIMERFUNCTION, self._set_timeout) self._multi.setopt(pycurl.M_SOCKETFUNCTION, self._handle_socket) self._curls = [self._curl_create() for i in range(max_clients)] self._free_list = self._curls[:] - self._requests = collections.deque() - self._fds = {} - self._timeout = None + self._requests = ( + collections.deque() + ) # type: Deque[Tuple[HTTPRequest, Callable[[HTTPResponse], None], float]] + self._fds = {} # type: Dict[int, int] + self._timeout = None # type: Optional[object] # libcurl has bugs that sometimes cause it to not report all # relevant file descriptors and timeouts to TIMERFUNCTION/ # SOCKETFUNCTION. Mitigate the effects of such bugs by # forcing a periodic scan of all active requests. self._force_timeout_callback = ioloop.PeriodicCallback( - self._handle_force_timeout, 1000) + self._handle_force_timeout, 1000 + ) self._force_timeout_callback.start() # Work around a bug in libcurl 7.29.0: Some fields in the curl @@ -64,7 +79,7 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): self._multi.add_handle(dummy_curl_handle) self._multi.remove_handle(dummy_curl_handle) - def close(self): + def close(self) -> None: self._force_timeout_callback.stop() if self._timeout is not None: self.io_loop.remove_timeout(self._timeout) @@ -76,15 +91,17 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): # Set below properties to None to reduce the reference count of current # instance, because those properties hold some methods of current # instance that will case circular reference. - self._force_timeout_callback = None + self._force_timeout_callback = None # type: ignore self._multi = None - def fetch_impl(self, request, callback): + def fetch_impl( + self, request: HTTPRequest, callback: Callable[[HTTPResponse], None] + ) -> None: self._requests.append((request, callback, self.io_loop.time())) self._process_queue() self._set_timeout(0) - def _handle_socket(self, event, fd, multi, data): + def _handle_socket(self, event: int, fd: int, multi: Any, data: bytes) -> None: """Called by libcurl when it wants to change the file descriptors it cares about. """ @@ -92,7 +109,7 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): pycurl.POLL_NONE: ioloop.IOLoop.NONE, pycurl.POLL_IN: ioloop.IOLoop.READ, pycurl.POLL_OUT: ioloop.IOLoop.WRITE, - pycurl.POLL_INOUT: ioloop.IOLoop.READ | ioloop.IOLoop.WRITE + pycurl.POLL_INOUT: ioloop.IOLoop.READ | ioloop.IOLoop.WRITE, } if event == pycurl.POLL_REMOVE: if fd in self._fds: @@ -110,18 +127,18 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): # instead of update. if fd in self._fds: self.io_loop.remove_handler(fd) - self.io_loop.add_handler(fd, self._handle_events, - ioloop_event) + self.io_loop.add_handler(fd, self._handle_events, ioloop_event) self._fds[fd] = ioloop_event - def _set_timeout(self, msecs): + def _set_timeout(self, msecs: int) -> None: """Called by libcurl to schedule a timeout.""" if self._timeout is not None: self.io_loop.remove_timeout(self._timeout) self._timeout = self.io_loop.add_timeout( - self.io_loop.time() + msecs / 1000.0, self._handle_timeout) + self.io_loop.time() + msecs / 1000.0, self._handle_timeout + ) - def _handle_events(self, fd, events): + def _handle_events(self, fd: int, events: int) -> None: """Called by IOLoop when there is activity on one of our file descriptors. """ @@ -139,19 +156,17 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): break self._finish_pending_requests() - def _handle_timeout(self): + def _handle_timeout(self) -> None: """Called by IOLoop when the requested timeout has passed.""" - with stack_context.NullContext(): - self._timeout = None - while True: - try: - ret, num_handles = self._multi.socket_action( - pycurl.SOCKET_TIMEOUT, 0) - except pycurl.error as e: - ret = e.args[0] - if ret != pycurl.E_CALL_MULTI_PERFORM: - break - self._finish_pending_requests() + self._timeout = None + while True: + try: + ret, num_handles = self._multi.socket_action(pycurl.SOCKET_TIMEOUT, 0) + except pycurl.error as e: + ret = e.args[0] + if ret != pycurl.E_CALL_MULTI_PERFORM: + break + self._finish_pending_requests() # In theory, we shouldn't have to do this because curl will # call _set_timeout whenever the timeout changes. However, @@ -170,21 +185,20 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): if new_timeout >= 0: self._set_timeout(new_timeout) - def _handle_force_timeout(self): + def _handle_force_timeout(self) -> None: """Called by IOLoop periodically to ask libcurl to process any events it may have forgotten about. """ - with stack_context.NullContext(): - while True: - try: - ret, num_handles = self._multi.socket_all() - except pycurl.error as e: - ret = e.args[0] - if ret != pycurl.E_CALL_MULTI_PERFORM: - break - self._finish_pending_requests() + while True: + try: + ret, num_handles = self._multi.socket_all() + except pycurl.error as e: + ret = e.args[0] + if ret != pycurl.E_CALL_MULTI_PERFORM: + break + self._finish_pending_requests() - def _finish_pending_requests(self): + def _finish_pending_requests(self) -> None: """Process any requests that were completed by the last call to multi.socket_action. """ @@ -198,55 +212,55 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): break self._process_queue() - def _process_queue(self): - with stack_context.NullContext(): - while True: - started = 0 - while self._free_list and self._requests: - started += 1 - curl = self._free_list.pop() - (request, callback, queue_start_time) = self._requests.popleft() - curl.info = { - "headers": httputil.HTTPHeaders(), - "buffer": BytesIO(), - "request": request, - "callback": callback, - "queue_start_time": queue_start_time, - "curl_start_time": time.time(), - "curl_start_ioloop_time": self.io_loop.current().time(), - } - try: - self._curl_setup_request( - curl, request, curl.info["buffer"], - curl.info["headers"]) - except Exception as e: - # If there was an error in setup, pass it on - # to the callback. Note that allowing the - # error to escape here will appear to work - # most of the time since we are still in the - # caller's original stack frame, but when - # _process_queue() is called from - # _finish_pending_requests the exceptions have - # nowhere to go. - self._free_list.append(curl) - callback(HTTPResponse( - request=request, - code=599, - error=e)) - else: - self._multi.add_handle(curl) + def _process_queue(self) -> None: + while True: + started = 0 + while self._free_list and self._requests: + started += 1 + curl = self._free_list.pop() + (request, callback, queue_start_time) = self._requests.popleft() + curl.info = { + "headers": httputil.HTTPHeaders(), + "buffer": BytesIO(), + "request": request, + "callback": callback, + "queue_start_time": queue_start_time, + "curl_start_time": time.time(), + "curl_start_ioloop_time": self.io_loop.current().time(), + } + try: + self._curl_setup_request( + curl, request, curl.info["buffer"], curl.info["headers"] + ) + except Exception as e: + # If there was an error in setup, pass it on + # to the callback. Note that allowing the + # error to escape here will appear to work + # most of the time since we are still in the + # caller's original stack frame, but when + # _process_queue() is called from + # _finish_pending_requests the exceptions have + # nowhere to go. + self._free_list.append(curl) + callback(HTTPResponse(request=request, code=599, error=e)) + else: + self._multi.add_handle(curl) - if not started: - break + if not started: + break - def _finish(self, curl, curl_error=None, curl_message=None): + def _finish( + self, curl: pycurl.Curl, curl_error: int = None, curl_message: str = None + ) -> None: info = curl.info curl.info = None self._multi.remove_handle(curl) self._free_list.append(curl) buffer = info["buffer"] if curl_error: - error = CurlError(curl_error, curl_message) + assert curl_message is not None + error = CurlError(curl_error, curl_message) # type: Optional[CurlError] + assert error is not None code = error.code effective_url = None buffer.close() @@ -269,30 +283,45 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): redirect=curl.getinfo(pycurl.REDIRECT_TIME), ) try: - info["callback"](HTTPResponse( - request=info["request"], code=code, headers=info["headers"], - buffer=buffer, effective_url=effective_url, error=error, - reason=info['headers'].get("X-Http-Reason", None), - request_time=self.io_loop.time() - info["curl_start_ioloop_time"], - start_time=info["curl_start_time"], - time_info=time_info)) + info["callback"]( + HTTPResponse( + request=info["request"], + code=code, + headers=info["headers"], + buffer=buffer, + effective_url=effective_url, + error=error, + reason=info["headers"].get("X-Http-Reason", None), + request_time=self.io_loop.time() - info["curl_start_ioloop_time"], + start_time=info["curl_start_time"], + time_info=time_info, + ) + ) except Exception: self.handle_callback_exception(info["callback"]) - def handle_callback_exception(self, callback): - self.io_loop.handle_callback_exception(callback) + def handle_callback_exception(self, callback: Any) -> None: + app_log.error("Exception in callback %r", callback, exc_info=True) - def _curl_create(self): + def _curl_create(self) -> pycurl.Curl: curl = pycurl.Curl() if curl_log.isEnabledFor(logging.DEBUG): curl.setopt(pycurl.VERBOSE, 1) curl.setopt(pycurl.DEBUGFUNCTION, self._curl_debug) - if hasattr(pycurl, 'PROTOCOLS'): # PROTOCOLS first appeared in pycurl 7.19.5 (2014-07-12) + if hasattr( + pycurl, "PROTOCOLS" + ): # PROTOCOLS first appeared in pycurl 7.19.5 (2014-07-12) curl.setopt(pycurl.PROTOCOLS, pycurl.PROTO_HTTP | pycurl.PROTO_HTTPS) curl.setopt(pycurl.REDIR_PROTOCOLS, pycurl.PROTO_HTTP | pycurl.PROTO_HTTPS) return curl - def _curl_setup_request(self, curl, request, buffer, headers): + def _curl_setup_request( + self, + curl: pycurl.Curl, + request: HTTPRequest, + buffer: BytesIO, + headers: httputil.HTTPHeaders, + ) -> None: curl.setopt(pycurl.URL, native_str(request.url)) # libcurl's magic "Expect: 100-continue" behavior causes delays @@ -310,22 +339,35 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): if "Pragma" not in request.headers: request.headers["Pragma"] = "" - curl.setopt(pycurl.HTTPHEADER, - ["%s: %s" % (native_str(k), native_str(v)) - for k, v in request.headers.get_all()]) + curl.setopt( + pycurl.HTTPHEADER, + [ + "%s: %s" % (native_str(k), native_str(v)) + for k, v in request.headers.get_all() + ], + ) - curl.setopt(pycurl.HEADERFUNCTION, - functools.partial(self._curl_header_callback, - headers, request.header_callback)) + curl.setopt( + pycurl.HEADERFUNCTION, + functools.partial( + self._curl_header_callback, headers, request.header_callback + ), + ) if request.streaming_callback: - def write_function(chunk): - self.io_loop.add_callback(request.streaming_callback, chunk) + + def write_function(b: Union[bytes, bytearray]) -> int: + assert request.streaming_callback is not None + self.io_loop.add_callback(request.streaming_callback, b) + return len(b) + else: write_function = buffer.write curl.setopt(pycurl.WRITEFUNCTION, write_function) curl.setopt(pycurl.FOLLOWLOCATION, request.follow_redirects) curl.setopt(pycurl.MAXREDIRS, request.max_redirects) + assert request.connect_timeout is not None curl.setopt(pycurl.CONNECTTIMEOUT_MS, int(1000 * request.connect_timeout)) + assert request.request_timeout is not None curl.setopt(pycurl.TIMEOUT_MS, int(1000 * request.request_timeout)) if request.user_agent: curl.setopt(pycurl.USERAGENT, native_str(request.user_agent)) @@ -341,20 +383,22 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): curl.setopt(pycurl.PROXY, request.proxy_host) curl.setopt(pycurl.PROXYPORT, request.proxy_port) if request.proxy_username: - credentials = httputil.encode_username_password(request.proxy_username, - request.proxy_password) + assert request.proxy_password is not None + credentials = httputil.encode_username_password( + request.proxy_username, request.proxy_password + ) curl.setopt(pycurl.PROXYUSERPWD, credentials) - if (request.proxy_auth_mode is None or - request.proxy_auth_mode == "basic"): + if request.proxy_auth_mode is None or request.proxy_auth_mode == "basic": curl.setopt(pycurl.PROXYAUTH, pycurl.HTTPAUTH_BASIC) elif request.proxy_auth_mode == "digest": curl.setopt(pycurl.PROXYAUTH, pycurl.HTTPAUTH_DIGEST) else: raise ValueError( - "Unsupported proxy_auth_mode %s" % request.proxy_auth_mode) + "Unsupported proxy_auth_mode %s" % request.proxy_auth_mode + ) else: - curl.setopt(pycurl.PROXY, '') + curl.setopt(pycurl.PROXY, "") curl.unsetopt(pycurl.PROXYUSERPWD) if request.validate_cert: curl.setopt(pycurl.SSL_VERIFYPEER, 1) @@ -397,7 +441,7 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): elif request.allow_nonstandard_methods or request.method in custom_methods: curl.setopt(pycurl.CUSTOMREQUEST, request.method) else: - raise KeyError('unknown method ' + request.method) + raise KeyError("unknown method " + request.method) body_expected = request.method in ("POST", "PATCH", "PUT") body_present = request.body is not None @@ -405,12 +449,14 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): # Some HTTP methods nearly always have bodies while others # almost never do. Fail in this case unless the user has # opted out of sanity checks with allow_nonstandard_methods. - if ((body_expected and not body_present) or - (body_present and not body_expected)): + if (body_expected and not body_present) or ( + body_present and not body_expected + ): raise ValueError( - 'Body must %sbe None for method %s (unless ' - 'allow_nonstandard_methods is true)' % - ('not ' if body_expected else '', request.method)) + "Body must %sbe None for method %s (unless " + "allow_nonstandard_methods is true)" + % ("not " if body_expected else "", request.method) + ) if body_expected or body_present: if request.method == "GET": @@ -419,21 +465,23 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): # unless we use CUSTOMREQUEST). While the spec doesn't # forbid clients from sending a body, it arguably # disallows the server from doing anything with them. - raise ValueError('Body must be None for GET request') - request_buffer = BytesIO(utf8(request.body or '')) + raise ValueError("Body must be None for GET request") + request_buffer = BytesIO(utf8(request.body or "")) - def ioctl(cmd): + def ioctl(cmd: int) -> None: if cmd == curl.IOCMD_RESTARTREAD: request_buffer.seek(0) + curl.setopt(pycurl.READFUNCTION, request_buffer.read) curl.setopt(pycurl.IOCTLFUNCTION, ioctl) if request.method == "POST": - curl.setopt(pycurl.POSTFIELDSIZE, len(request.body or '')) + curl.setopt(pycurl.POSTFIELDSIZE, len(request.body or "")) else: curl.setopt(pycurl.UPLOAD, True) - curl.setopt(pycurl.INFILESIZE, len(request.body or '')) + curl.setopt(pycurl.INFILESIZE, len(request.body or "")) if request.auth_username is not None: + assert request.auth_password is not None if request.auth_mode is None or request.auth_mode == "basic": curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC) elif request.auth_mode == "digest": @@ -441,11 +489,16 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): else: raise ValueError("Unsupported auth_mode %s" % request.auth_mode) - userpwd = httputil.encode_username_password(request.auth_username, - request.auth_password) + userpwd = httputil.encode_username_password( + request.auth_username, request.auth_password + ) curl.setopt(pycurl.USERPWD, userpwd) - curl_log.debug("%s %s (username: %r)", request.method, request.url, - request.auth_username) + curl_log.debug( + "%s %s (username: %r)", + request.method, + request.url, + request.auth_username, + ) else: curl.unsetopt(pycurl.USERPWD) curl_log.debug("%s %s", request.method, request.url) @@ -459,7 +512,7 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): if request.ssl_options is not None: raise ValueError("ssl_options not supported in curl_httpclient") - if threading.activeCount() > 1: + if threading.active_count() > 1: # libcurl/pycurl is not thread-safe by default. When multiple threads # are used, signals should be disabled. This has the side effect # of disabling DNS timeouts in some environments (when libcurl is @@ -472,8 +525,13 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): if request.prepare_curl_callback is not None: request.prepare_curl_callback(curl) - def _curl_header_callback(self, headers, header_callback, header_line): - header_line = native_str(header_line.decode('latin1')) + def _curl_header_callback( + self, + headers: httputil.HTTPHeaders, + header_callback: Callable[[str], None], + header_line_bytes: bytes, + ) -> None: + header_line = native_str(header_line_bytes.decode("latin1")) if header_callback is not None: self.io_loop.add_callback(header_callback, header_line) # header_line as returned by curl includes the end-of-line characters. @@ -490,21 +548,21 @@ class CurlAsyncHTTPClient(AsyncHTTPClient): return headers.parse_line(header_line) - def _curl_debug(self, debug_type, debug_msg): - debug_types = ('I', '<', '>', '<', '>') + def _curl_debug(self, debug_type: int, debug_msg: str) -> None: + debug_types = ("I", "<", ">", "<", ">") if debug_type == 0: debug_msg = native_str(debug_msg) - curl_log.debug('%s', debug_msg.strip()) + curl_log.debug("%s", debug_msg.strip()) elif debug_type in (1, 2): debug_msg = native_str(debug_msg) for line in debug_msg.splitlines(): - curl_log.debug('%s %s', debug_types[debug_type], line) + curl_log.debug("%s %s", debug_types[debug_type], line) elif debug_type == 4: - curl_log.debug('%s %r', debug_types[debug_type], debug_msg) + curl_log.debug("%s %r", debug_types[debug_type], debug_msg) class CurlError(HTTPError): - def __init__(self, errno, message): + def __init__(self, errno: int, message: str) -> None: HTTPError.__init__(self, 599, message) self.errno = errno diff --git a/server/www/packages/packages-linux/x64/tornado/escape.py b/server/www/packages/packages-linux/x64/tornado/escape.py index a79ece6..b0ec332 100644 --- a/server/www/packages/packages-linux/x64/tornado/escape.py +++ b/server/www/packages/packages-linux/x64/tornado/escape.py @@ -19,35 +19,28 @@ Also includes a few other miscellaneous string manipulation functions that have crept in over time. """ -from __future__ import absolute_import, division, print_function - +import html.entities import json import re +import urllib.parse -from tornado.util import PY3, unicode_type, basestring_type +from tornado.util import unicode_type -if PY3: - from urllib.parse import parse_qs as _parse_qs - import html.entities as htmlentitydefs - import urllib.parse as urllib_parse - unichr = chr -else: - from urlparse import parse_qs as _parse_qs - import htmlentitydefs - import urllib as urllib_parse - -try: - import typing # noqa -except ImportError: - pass +import typing +from typing import Union, Any, Optional, Dict, List, Callable -_XHTML_ESCAPE_RE = re.compile('[&<>"\']') -_XHTML_ESCAPE_DICT = {'&': '&', '<': '<', '>': '>', '"': '"', - '\'': '''} +_XHTML_ESCAPE_RE = re.compile("[&<>\"']") +_XHTML_ESCAPE_DICT = { + "&": "&", + "<": "<", + ">": ">", + '"': """, + "'": "'", +} -def xhtml_escape(value): +def xhtml_escape(value: Union[str, bytes]) -> str: """Escapes a string so it is valid within HTML or XML. Escapes the characters ``<``, ``>``, ``"``, ``'``, and ``&``. @@ -58,11 +51,12 @@ def xhtml_escape(value): Added the single quote to the list of escaped characters. """ - return _XHTML_ESCAPE_RE.sub(lambda match: _XHTML_ESCAPE_DICT[match.group(0)], - to_basestring(value)) + return _XHTML_ESCAPE_RE.sub( + lambda match: _XHTML_ESCAPE_DICT[match.group(0)], to_basestring(value) + ) -def xhtml_unescape(value): +def xhtml_unescape(value: Union[str, bytes]) -> str: """Un-escapes an XML-escaped string.""" return re.sub(r"&(#?)(\w+?);", _convert_entity, _unicode(value)) @@ -70,7 +64,7 @@ def xhtml_unescape(value): # The fact that json_encode wraps json.dumps is an implementation detail. # Please see https://github.com/tornadoweb/tornado/pull/706 # before sending a pull request that adds **kwargs to this function. -def json_encode(value): +def json_encode(value: Any) -> str: """JSON-encodes the given Python object.""" # JSON permits but does not require forward slashes to be escaped. # This is useful when json data is emitted in a ' - for p in paths) + return "".join( + '' + for p in paths + ) - def render_embed_js(self, js_embed): + def render_embed_js(self, js_embed: Iterable[bytes]) -> bytes: """Default method used to render the final embedded js for the rendered webpage. Override this method in a sub-classed controller to change the output. """ - return b'' + return ( + b'" + ) - def render_linked_css(self, css_files): + def render_linked_css(self, css_files: Iterable[str]) -> str: """Default method used to render the final css links for the rendered webpage. Override this method in a sub-classed controller to change the output. """ paths = [] - unique_paths = set() + unique_paths = set() # type: Set[str] for path in css_files: if not is_absolute(path): @@ -868,20 +964,21 @@ class RequestHandler(object): paths.append(path) unique_paths.add(path) - return ''.join('' - for p in paths) + return "".join( + '' + for p in paths + ) - def render_embed_css(self, css_embed): + def render_embed_css(self, css_embed: Iterable[bytes]) -> bytes: """Default method used to render the final embedded css for the rendered webpage. Override this method in a sub-classed controller to change the output. """ - return b'' + return b'" - def render_string(self, template_name, **kwargs): + def render_string(self, template_name: str, **kwargs: Any) -> bytes: """Generate the given template with the given arguments. We return the generated byte string (in utf8). To generate and @@ -894,6 +991,7 @@ class RequestHandler(object): web_file = frame.f_code.co_filename while frame.f_code.co_filename == web_file: frame = frame.f_back + assert frame.f_code.co_filename is not None template_path = os.path.dirname(frame.f_code.co_filename) with RequestHandler._template_loader_lock: if template_path not in RequestHandler._template_loaders: @@ -906,7 +1004,7 @@ class RequestHandler(object): namespace.update(kwargs) return t.generate(**namespace) - def get_template_namespace(self): + def get_template_namespace(self) -> Dict[str, Any]: """Returns a dictionary to be used as the default template namespace. May be overridden by subclasses to add or modify values. @@ -924,12 +1022,12 @@ class RequestHandler(object): pgettext=self.locale.pgettext, static_url=self.static_url, xsrf_form_html=self.xsrf_form_html, - reverse_url=self.reverse_url + reverse_url=self.reverse_url, ) namespace.update(self.ui) return namespace - def create_template_loader(self, template_path): + def create_template_loader(self, template_path: str) -> template.BaseLoader: """Returns a new template loader for the given path. May be overridden by subclasses. By default returns a @@ -950,7 +1048,7 @@ class RequestHandler(object): kwargs["whitespace"] = settings["template_whitespace"] return template.Loader(template_path, **kwargs) - def flush(self, include_footers=False, callback=None): + def flush(self, include_footers: bool = False) -> "Future[None]": """Flushes the current output buffer to the network. The ``callback`` argument, if given, can be used for flow control: @@ -962,23 +1060,23 @@ class RequestHandler(object): .. versionchanged:: 4.0 Now returns a `.Future` if no callback is given. - .. deprecated:: 5.1 + .. versionchanged:: 6.0 - The ``callback`` argument is deprecated and will be removed in - Tornado 6.0. + The ``callback`` argument was removed. """ + assert self.request.connection is not None chunk = b"".join(self._write_buffer) self._write_buffer = [] if not self._headers_written: self._headers_written = True for transform in self._transforms: - self._status_code, self._headers, chunk = \ - transform.transform_first_chunk( - self._status_code, self._headers, - chunk, include_footers) + assert chunk is not None + self._status_code, self._headers, chunk = transform.transform_first_chunk( + self._status_code, self._headers, chunk, include_footers + ) # Ignore the chunk and only write the headers for HEAD requests if self.request.method == "HEAD": - chunk = None + chunk = b"" # Finalize the cookie headers (which have been stored in a side # object so an outgoing cookie could be overwritten before it @@ -987,23 +1085,22 @@ class RequestHandler(object): for cookie in self._new_cookie.values(): self.add_header("Set-Cookie", cookie.OutputString(None)) - start_line = httputil.ResponseStartLine('', - self._status_code, - self._reason) + start_line = httputil.ResponseStartLine("", self._status_code, self._reason) return self.request.connection.write_headers( - start_line, self._headers, chunk, callback=callback) + start_line, self._headers, chunk + ) else: for transform in self._transforms: chunk = transform.transform_chunk(chunk, include_footers) # Ignore the chunk and only write the headers for HEAD requests if self.request.method != "HEAD": - return self.request.connection.write(chunk, callback=callback) + return self.request.connection.write(chunk) else: - future = Future() + future = Future() # type: Future[None] future.set_result(None) return future - def finish(self, chunk=None): + def finish(self, chunk: Union[str, bytes, dict] = None) -> "Future[None]": """Finishes this response, ending the HTTP request. Passing a ``chunk`` to ``finish()`` is equivalent to passing that @@ -1027,27 +1124,32 @@ class RequestHandler(object): # Automatically support ETags and add the Content-Length header if # we have not flushed any content yet. if not self._headers_written: - if (self._status_code == 200 and - self.request.method in ("GET", "HEAD") and - "Etag" not in self._headers): + if ( + self._status_code == 200 + and self.request.method in ("GET", "HEAD") + and "Etag" not in self._headers + ): self.set_etag_header() if self.check_etag_header(): self._write_buffer = [] self.set_status(304) - if (self._status_code in (204, 304) or - (self._status_code >= 100 and self._status_code < 200)): - assert not self._write_buffer, "Cannot send body with %s" % self._status_code + if self._status_code in (204, 304) or ( + self._status_code >= 100 and self._status_code < 200 + ): + assert not self._write_buffer, ( + "Cannot send body with %s" % self._status_code + ) self._clear_headers_for_304() elif "Content-Length" not in self._headers: content_length = sum(len(part) for part in self._write_buffer) self.set_header("Content-Length", content_length) - if hasattr(self.request, "connection"): - # Now that the request is finished, clear the callback we - # set on the HTTPConnection (which would otherwise prevent the - # garbage collection of the RequestHandler when there - # are keepalive connections) - self.request.connection.set_close_callback(None) + assert self.request.connection is not None + # Now that the request is finished, clear the callback we + # set on the HTTPConnection (which would otherwise prevent the + # garbage collection of the RequestHandler when there + # are keepalive connections) + self.request.connection.set_close_callback(None) # type: ignore future = self.flush(include_footers=True) self.request.connection.finish() @@ -1057,7 +1159,7 @@ class RequestHandler(object): self._break_cycles() return future - def detach(self): + def detach(self) -> iostream.IOStream: """Take control of the underlying stream. Returns the underlying `.IOStream` object and stops all @@ -1069,14 +1171,15 @@ class RequestHandler(object): .. versionadded:: 5.1 """ self._finished = True - return self.request.connection.detach() + # TODO: add detach to HTTPConnection? + return self.request.connection.detach() # type: ignore - def _break_cycles(self): + def _break_cycles(self) -> None: # Break up a reference cycle between this handler and the # _ui_module closures to allow for faster GC on CPython. - self.ui = None + self.ui = None # type: ignore - def send_error(self, status_code=500, **kwargs): + def send_error(self, status_code: int = 500, **kwargs: Any) -> None: """Sends the given HTTP error code to the browser. If `flush()` has already been called, it is not possible to send @@ -1097,14 +1200,13 @@ class RequestHandler(object): try: self.finish() except Exception: - gen_log.error("Failed to flush partial response", - exc_info=True) + gen_log.error("Failed to flush partial response", exc_info=True) return self.clear() - reason = kwargs.get('reason') - if 'exc_info' in kwargs: - exception = kwargs['exc_info'][1] + reason = kwargs.get("reason") + if "exc_info" in kwargs: + exception = kwargs["exc_info"][1] if isinstance(exception, HTTPError) and exception.reason: reason = exception.reason self.set_status(status_code, reason=reason) @@ -1115,7 +1217,7 @@ class RequestHandler(object): if not self._finished: self.finish() - def write_error(self, status_code, **kwargs): + def write_error(self, status_code: int, **kwargs: Any) -> None: """Override to implement custom error pages. ``write_error`` may call `write`, `render`, `set_header`, etc @@ -1129,19 +1231,19 @@ class RequestHandler(object): """ if self.settings.get("serve_traceback") and "exc_info" in kwargs: # in debug mode, try to send a traceback - self.set_header('Content-Type', 'text/plain') + self.set_header("Content-Type", "text/plain") for line in traceback.format_exception(*kwargs["exc_info"]): self.write(line) self.finish() else: - self.finish("%(code)d: %(message)s" - "%(code)d: %(message)s" % { - "code": status_code, - "message": self._reason, - }) + self.finish( + "%(code)d: %(message)s" + "%(code)d: %(message)s" + % {"code": status_code, "message": self._reason} + ) @property - def locale(self): + def locale(self) -> tornado.locale.Locale: """The locale for the current session. Determined by either `get_user_locale`, which you can override to @@ -1153,17 +1255,19 @@ class RequestHandler(object): Added a property setter. """ if not hasattr(self, "_locale"): - self._locale = self.get_user_locale() - if not self._locale: + loc = self.get_user_locale() + if loc is not None: + self._locale = loc + else: self._locale = self.get_browser_locale() assert self._locale return self._locale @locale.setter - def locale(self, value): + def locale(self, value: tornado.locale.Locale) -> None: self._locale = value - def get_user_locale(self): + def get_user_locale(self) -> Optional[tornado.locale.Locale]: """Override to determine the locale from the authenticated user. If None is returned, we fall back to `get_browser_locale()`. @@ -1173,7 +1277,7 @@ class RequestHandler(object): """ return None - def get_browser_locale(self, default="en_US"): + def get_browser_locale(self, default: str = "en_US") -> tornado.locale.Locale: """Determines the user's locale from ``Accept-Language`` header. See http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.4 @@ -1198,7 +1302,7 @@ class RequestHandler(object): return locale.get(default) @property - def current_user(self): + def current_user(self) -> Any: """The authenticated user for this request. This is set in one of two ways: @@ -1234,17 +1338,17 @@ class RequestHandler(object): return self._current_user @current_user.setter - def current_user(self, value): + def current_user(self, value: Any) -> None: self._current_user = value - def get_current_user(self): + def get_current_user(self) -> Any: """Override to determine the current user from, e.g., a cookie. This method may not be a coroutine. """ return None - def get_login_url(self): + def get_login_url(self) -> str: """Override to customize the login URL based on the request. By default, we use the ``login_url`` application setting. @@ -1252,7 +1356,7 @@ class RequestHandler(object): self.require_setting("login_url", "@tornado.web.authenticated") return self.application.settings["login_url"] - def get_template_path(self): + def get_template_path(self) -> Optional[str]: """Override to customize template path for each handler. By default, we use the ``template_path`` application setting. @@ -1261,7 +1365,7 @@ class RequestHandler(object): return self.application.settings.get("template_path") @property - def xsrf_token(self): + def xsrf_token(self) -> bytes: """The XSRF-prevention token for the current user/session. To prevent cross-site request forgery, we set an '_xsrf' cookie @@ -1301,22 +1405,23 @@ class RequestHandler(object): self._xsrf_token = binascii.b2a_hex(token) elif output_version == 2: mask = os.urandom(4) - self._xsrf_token = b"|".join([ - b"2", - binascii.b2a_hex(mask), - binascii.b2a_hex(_websocket_mask(mask, token)), - utf8(str(int(timestamp)))]) + self._xsrf_token = b"|".join( + [ + b"2", + binascii.b2a_hex(mask), + binascii.b2a_hex(_websocket_mask(mask, token)), + utf8(str(int(timestamp))), + ] + ) else: - raise ValueError("unknown xsrf cookie version %d", - output_version) + raise ValueError("unknown xsrf cookie version %d", output_version) if version is None: - expires_days = 30 if self.current_user else None - self.set_cookie("_xsrf", self._xsrf_token, - expires_days=expires_days, - **cookie_kwargs) + if self.current_user and "expires_days" not in cookie_kwargs: + cookie_kwargs["expires_days"] = 30 + self.set_cookie("_xsrf", self._xsrf_token, **cookie_kwargs) return self._xsrf_token - def _get_raw_xsrf_token(self): + def _get_raw_xsrf_token(self) -> Tuple[Optional[int], bytes, float]: """Read or generate the xsrf token in its raw form. The raw_xsrf_token is a tuple containing: @@ -1327,7 +1432,7 @@ class RequestHandler(object): * timestamp: the time this token was generated (will not be accurate for version 1 cookies) """ - if not hasattr(self, '_raw_xsrf_token'): + if not hasattr(self, "_raw_xsrf_token"): cookie = self.get_cookie("_xsrf") if cookie: version, token, timestamp = self._decode_xsrf_token(cookie) @@ -1337,10 +1442,14 @@ class RequestHandler(object): version = None token = os.urandom(16) timestamp = time.time() + assert token is not None + assert timestamp is not None self._raw_xsrf_token = (version, token, timestamp) return self._raw_xsrf_token - def _decode_xsrf_token(self, cookie): + def _decode_xsrf_token( + self, cookie: str + ) -> Tuple[Optional[int], Optional[bytes], Optional[float]]: """Convert a cookie string into a the tuple form returned by _get_raw_xsrf_token. """ @@ -1351,12 +1460,11 @@ class RequestHandler(object): if m: version = int(m.group(1)) if version == 2: - _, mask, masked_token, timestamp = cookie.split("|") + _, mask_str, masked_token, timestamp_str = cookie.split("|") - mask = binascii.a2b_hex(utf8(mask)) - token = _websocket_mask( - mask, binascii.a2b_hex(utf8(masked_token))) - timestamp = int(timestamp) + mask = binascii.a2b_hex(utf8(mask_str)) + token = _websocket_mask(mask, binascii.a2b_hex(utf8(masked_token))) + timestamp = int(timestamp_str) return version, token, timestamp else: # Treat unknown versions as not present instead of failing. @@ -1372,11 +1480,10 @@ class RequestHandler(object): return (version, token, timestamp) except Exception: # Catch exceptions and return nothing instead of failing. - gen_log.debug("Uncaught exception in _decode_xsrf_token", - exc_info=True) + gen_log.debug("Uncaught exception in _decode_xsrf_token", exc_info=True) return None, None, None - def check_xsrf_cookie(self): + def check_xsrf_cookie(self) -> None: """Verifies that the ``_xsrf`` cookie matches the ``_xsrf`` argument. To prevent cross-site request forgery, we set an ``_xsrf`` @@ -1390,30 +1497,31 @@ class RequestHandler(object): See http://en.wikipedia.org/wiki/Cross-site_request_forgery - Prior to release 1.1.1, this check was ignored if the HTTP header - ``X-Requested-With: XMLHTTPRequest`` was present. This exception - has been shown to be insecure and has been removed. For more - information please see - http://www.djangoproject.com/weblog/2011/feb/08/security/ - http://weblog.rubyonrails.org/2011/2/8/csrf-protection-bypass-in-ruby-on-rails - .. versionchanged:: 3.2.2 Added support for cookie version 2. Both versions 1 and 2 are supported. """ - token = (self.get_argument("_xsrf", None) or - self.request.headers.get("X-Xsrftoken") or - self.request.headers.get("X-Csrftoken")) + # Prior to release 1.1.1, this check was ignored if the HTTP header + # ``X-Requested-With: XMLHTTPRequest`` was present. This exception + # has been shown to be insecure and has been removed. For more + # information please see + # http://www.djangoproject.com/weblog/2011/feb/08/security/ + # http://weblog.rubyonrails.org/2011/2/8/csrf-protection-bypass-in-ruby-on-rails + token = ( + self.get_argument("_xsrf", None) + or self.request.headers.get("X-Xsrftoken") + or self.request.headers.get("X-Csrftoken") + ) if not token: raise HTTPError(403, "'_xsrf' argument missing from POST") _, token, _ = self._decode_xsrf_token(token) _, expected_token, _ = self._get_raw_xsrf_token() if not token: raise HTTPError(403, "'_xsrf' argument has invalid format") - if not _time_independent_equals(utf8(token), utf8(expected_token)): + if not hmac.compare_digest(utf8(token), utf8(expected_token)): raise HTTPError(403, "XSRF cookie does not match POST argument") - def xsrf_form_html(self): + def xsrf_form_html(self) -> str: """An HTML ```` element to be included with all POST forms. It defines the ``_xsrf`` input value, which we check on all POST @@ -1426,10 +1534,13 @@ class RequestHandler(object): See `check_xsrf_cookie()` above for more information. """ - return '' + return ( + '' + ) - def static_url(self, path, include_host=None, **kwargs): + def static_url(self, path: str, include_host: bool = None, **kwargs: Any) -> str: """Returns a static URL for the given relative static file path. This method requires you set the ``static_path`` setting in your @@ -1451,8 +1562,9 @@ class RequestHandler(object): """ self.require_setting("static_path", "static_url") - get_url = self.settings.get("static_handler_class", - StaticFileHandler).make_static_url + get_url = self.settings.get( + "static_handler_class", StaticFileHandler + ).make_static_url if include_host is None: include_host = getattr(self, "include_host", False) @@ -1464,17 +1576,19 @@ class RequestHandler(object): return base + get_url(self.settings, path, **kwargs) - def require_setting(self, name, feature="this feature"): + def require_setting(self, name: str, feature: str = "this feature") -> None: """Raises an exception if the given app setting is not defined.""" if not self.application.settings.get(name): - raise Exception("You must define the '%s' setting in your " - "application to use %s" % (name, feature)) + raise Exception( + "You must define the '%s' setting in your " + "application to use %s" % (name, feature) + ) - def reverse_url(self, name, *args): + def reverse_url(self, name: str, *args: Any) -> str: """Alias for `Application.reverse_url`.""" return self.application.reverse_url(name, *args) - def compute_etag(self): + def compute_etag(self) -> Optional[str]: """Computes the etag header to be used for this request. By default uses a hash of the content written so far. @@ -1487,7 +1601,7 @@ class RequestHandler(object): hasher.update(part) return '"%s"' % hasher.hexdigest() - def set_etag_header(self): + def set_etag_header(self) -> None: """Sets the response's Etag header using ``self.compute_etag()``. Note: no header will be set if ``compute_etag()`` returns ``None``. @@ -1498,7 +1612,7 @@ class RequestHandler(object): if etag is not None: self.set_header("Etag", etag) - def check_etag_header(self): + def check_etag_header(self) -> bool: """Checks the ``Etag`` header against requests's ``If-None-Match``. Returns ``True`` if the request's Etag matches and a 304 should be @@ -1519,19 +1633,18 @@ class RequestHandler(object): # Find all weak and strong etag values from If-None-Match header # because RFC 7232 allows multiple etag values in a single header. etags = re.findall( - br'\*|(?:W/)?"[^"]*"', - utf8(self.request.headers.get("If-None-Match", "")) + br'\*|(?:W/)?"[^"]*"', utf8(self.request.headers.get("If-None-Match", "")) ) if not computed_etag or not etags: return False match = False - if etags[0] == b'*': + if etags[0] == b"*": match = True else: # Use a weak comparison when comparing entity-tags. - def val(x): - return x[2:] if x.startswith(b'W/') else x + def val(x: bytes) -> bytes: + return x[2:] if x.startswith(b"W/") else x for etag in etags: if val(etag) == val(computed_etag): @@ -1539,36 +1652,30 @@ class RequestHandler(object): break return match - def _stack_context_handle_exception(self, type, value, traceback): - try: - # For historical reasons _handle_request_exception only takes - # the exception value instead of the full triple, - # so re-raise the exception to ensure that it's in - # sys.exc_info() - raise_exc_info((type, value, traceback)) - except Exception: - self._handle_request_exception(value) - return True - - @gen.coroutine - def _execute(self, transforms, *args, **kwargs): + async def _execute( + self, transforms: List["OutputTransform"], *args: bytes, **kwargs: bytes + ) -> None: """Executes this request with the given output transforms.""" self._transforms = transforms try: if self.request.method not in self.SUPPORTED_METHODS: raise HTTPError(405) self.path_args = [self.decode_argument(arg) for arg in args] - self.path_kwargs = dict((k, self.decode_argument(v, name=k)) - for (k, v) in kwargs.items()) + self.path_kwargs = dict( + (k, self.decode_argument(v, name=k)) for (k, v) in kwargs.items() + ) # If XSRF cookies are turned on, reject form submissions without # the proper cookie - if self.request.method not in ("GET", "HEAD", "OPTIONS") and \ - self.application.settings.get("xsrf_cookies"): + if self.request.method not in ( + "GET", + "HEAD", + "OPTIONS", + ) and self.application.settings.get("xsrf_cookies"): self.check_xsrf_cookie() result = self.prepare() if result is not None: - result = yield result + result = await result if self._prepared_future is not None: # Tell the Application we've finished with prepare() # and are ready for the body to arrive. @@ -1582,14 +1689,14 @@ class RequestHandler(object): # result; the data has been passed to self.data_received # instead. try: - yield self.request.body + await self.request._body_future except iostream.StreamClosedError: return method = getattr(self, self.request.method.lower()) result = method(*self.path_args, **self.path_kwargs) if result is not None: - result = yield result + result = await result if self._auto_finish and not self._finished: self.finish() except Exception as e: @@ -1600,21 +1707,22 @@ class RequestHandler(object): finally: # Unset result to avoid circular references result = None - if (self._prepared_future is not None and - not self._prepared_future.done()): + if self._prepared_future is not None and not self._prepared_future.done(): # In case we failed before setting _prepared_future, do it # now (to unblock the HTTP server). Note that this is not # in a finally block to avoid GC issues prior to Python 3.4. self._prepared_future.set_result(None) - def data_received(self, chunk): + def data_received(self, chunk: bytes) -> Optional[Awaitable[None]]: """Implement this method to handle streamed request data. Requires the `.stream_request_body` decorator. + + May be a coroutine for flow control. """ raise NotImplementedError() - def _log(self): + def _log(self) -> None: """Logs the current request. Sort of deprecated since this functionality was moved to the @@ -1623,11 +1731,14 @@ class RequestHandler(object): """ self.application.log_request(self) - def _request_summary(self): - return "%s %s (%s)" % (self.request.method, self.request.uri, - self.request.remote_ip) + def _request_summary(self) -> str: + return "%s %s (%s)" % ( + self.request.method, + self.request.uri, + self.request.remote_ip, + ) - def _handle_request_exception(self, e): + def _handle_request_exception(self, e: BaseException) -> None: if isinstance(e, Finish): # Not an error; just finish the request without logging. if not self._finished: @@ -1649,7 +1760,12 @@ class RequestHandler(object): else: self.send_error(500, exc_info=sys.exc_info()) - def log_exception(self, typ, value, tb): + def log_exception( + self, + typ: "Optional[Type[BaseException]]", + value: Optional[BaseException], + tb: Optional[TracebackType], + ) -> None: """Override to customize logging of uncaught exceptions. By default logs instances of `HTTPError` as warnings without @@ -1662,123 +1778,50 @@ class RequestHandler(object): if isinstance(value, HTTPError): if value.log_message: format = "%d %s: " + value.log_message - args = ([value.status_code, self._request_summary()] + - list(value.args)) + args = [value.status_code, self._request_summary()] + list(value.args) gen_log.warning(format, *args) else: - app_log.error("Uncaught exception %s\n%r", self._request_summary(), - self.request, exc_info=(typ, value, tb)) + app_log.error( # type: ignore + "Uncaught exception %s\n%r", + self._request_summary(), + self.request, + exc_info=(typ, value, tb), + ) - def _ui_module(self, name, module): - def render(*args, **kwargs): + def _ui_module(self, name: str, module: Type["UIModule"]) -> Callable[..., str]: + def render(*args, **kwargs) -> str: # type: ignore if not hasattr(self, "_active_modules"): - self._active_modules = {} + self._active_modules = {} # type: Dict[str, UIModule] if name not in self._active_modules: self._active_modules[name] = module(self) rendered = self._active_modules[name].render(*args, **kwargs) return rendered + return render - def _ui_method(self, method): + def _ui_method(self, method: Callable[..., str]) -> Callable[..., str]: return lambda *args, **kwargs: method(self, *args, **kwargs) - def _clear_headers_for_304(self): + def _clear_headers_for_304(self) -> None: # 304 responses should not contain entity headers (defined in # http://www.w3.org/Protocols/rfc2616/rfc2616-sec7.html#sec7.1) # not explicitly allowed by # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.5 - headers = ["Allow", "Content-Encoding", "Content-Language", - "Content-Length", "Content-MD5", "Content-Range", - "Content-Type", "Last-Modified"] + headers = [ + "Allow", + "Content-Encoding", + "Content-Language", + "Content-Length", + "Content-MD5", + "Content-Range", + "Content-Type", + "Last-Modified", + ] for h in headers: self.clear_header(h) -def asynchronous(method): - """Wrap request handler methods with this if they are asynchronous. - - This decorator is for callback-style asynchronous methods; for - coroutines, use the ``@gen.coroutine`` decorator without - ``@asynchronous``. (It is legal for legacy reasons to use the two - decorators together provided ``@asynchronous`` is first, but - ``@asynchronous`` will be ignored in this case) - - This decorator should only be applied to the :ref:`HTTP verb - methods `; its behavior is undefined for any other method. - This decorator does not *make* a method asynchronous; it tells - the framework that the method *is* asynchronous. For this decorator - to be useful the method must (at least sometimes) do something - asynchronous. - - If this decorator is given, the response is not finished when the - method returns. It is up to the request handler to call - `self.finish() ` to finish the HTTP - request. Without this decorator, the request is automatically - finished when the ``get()`` or ``post()`` method returns. Example: - - .. testcode:: - - class MyRequestHandler(RequestHandler): - @asynchronous - def get(self): - http = httpclient.AsyncHTTPClient() - http.fetch("http://friendfeed.com/", self._on_download) - - def _on_download(self, response): - self.write("Downloaded!") - self.finish() - - .. testoutput:: - :hide: - - .. versionchanged:: 3.1 - The ability to use ``@gen.coroutine`` without ``@asynchronous``. - - .. versionchanged:: 4.3 Returning anything but ``None`` or a - yieldable object from a method decorated with ``@asynchronous`` - is an error. Such return values were previously ignored silently. - - .. deprecated:: 5.1 - - This decorator is deprecated and will be removed in Tornado 6.0. - Use coroutines instead. - """ - warnings.warn("@asynchronous is deprecated, use coroutines instead", - DeprecationWarning) - # Delay the IOLoop import because it's not available on app engine. - from tornado.ioloop import IOLoop - - @functools.wraps(method) - def wrapper(self, *args, **kwargs): - self._auto_finish = False - with stack_context.ExceptionStackContext( - self._stack_context_handle_exception, delay_warning=True): - result = method(self, *args, **kwargs) - if result is not None: - result = gen.convert_yielded(result) - - # If @asynchronous is used with @gen.coroutine, (but - # not @gen.engine), we can automatically finish the - # request when the future resolves. Additionally, - # the Future will swallow any exceptions so we need - # to throw them back out to the stack context to finish - # the request. - def future_complete(f): - f.result() - if not self._finished: - self.finish() - IOLoop.current().add_future(result, future_complete) - # Once we have done this, hide the Future from our - # caller (i.e. RequestHandler._when_complete), which - # would otherwise set up its own callback and - # exception handler (resulting in exceptions being - # logged twice). - return None - return result - return wrapper - - -def stream_request_body(cls): +def stream_request_body(cls: Type[RequestHandler]) -> Type[RequestHandler]: """Apply to `RequestHandler` subclasses to enable streaming body support. This decorator implies the following changes: @@ -1805,21 +1848,26 @@ def stream_request_body(cls): return cls -def _has_stream_request_body(cls): +def _has_stream_request_body(cls: Type[RequestHandler]) -> bool: if not issubclass(cls, RequestHandler): raise TypeError("expected subclass of RequestHandler, got %r", cls) - return getattr(cls, '_stream_request_body', False) + return cls._stream_request_body -def removeslash(method): +def removeslash( + method: Callable[..., Optional[Awaitable[None]]] +) -> Callable[..., Optional[Awaitable[None]]]: """Use this decorator to remove trailing slashes from the request path. For example, a request to ``/foo/`` would redirect to ``/foo`` with this decorator. Your request handler mapping should use a regular expression like ``r'/foo/*'`` in conjunction with using the decorator. """ + @functools.wraps(method) - def wrapper(self, *args, **kwargs): + def wrapper( # type: ignore + self: RequestHandler, *args, **kwargs + ) -> Optional[Awaitable[None]]: if self.request.path.endswith("/"): if self.request.method in ("GET", "HEAD"): uri = self.request.path.rstrip("/") @@ -1827,31 +1875,38 @@ def removeslash(method): if self.request.query: uri += "?" + self.request.query self.redirect(uri, permanent=True) - return + return None else: raise HTTPError(404) return method(self, *args, **kwargs) + return wrapper -def addslash(method): +def addslash( + method: Callable[..., Optional[Awaitable[None]]] +) -> Callable[..., Optional[Awaitable[None]]]: """Use this decorator to add a missing trailing slash to the request path. For example, a request to ``/foo`` would redirect to ``/foo/`` with this decorator. Your request handler mapping should use a regular expression like ``r'/foo/?'`` in conjunction with using the decorator. """ + @functools.wraps(method) - def wrapper(self, *args, **kwargs): + def wrapper( # type: ignore + self: RequestHandler, *args, **kwargs + ) -> Optional[Awaitable[None]]: if not self.request.path.endswith("/"): if self.request.method in ("GET", "HEAD"): uri = self.request.path + "/" if self.request.query: uri += "?" + self.request.query self.redirect(uri, permanent=True) - return + return None raise HTTPError(404) return method(self, *args, **kwargs) + return wrapper @@ -1866,28 +1921,36 @@ class _ApplicationRouter(ReversibleRuleRouter): `_ApplicationRouter` instance. """ - def __init__(self, application, rules=None): + def __init__(self, application: "Application", rules: _RuleList = None) -> None: assert isinstance(application, Application) self.application = application super(_ApplicationRouter, self).__init__(rules) - def process_rule(self, rule): + def process_rule(self, rule: Rule) -> Rule: rule = super(_ApplicationRouter, self).process_rule(rule) if isinstance(rule.target, (list, tuple)): - rule.target = _ApplicationRouter(self.application, rule.target) + rule.target = _ApplicationRouter( # type: ignore + self.application, rule.target + ) return rule - def get_target_delegate(self, target, request, **target_params): + def get_target_delegate( + self, target: Any, request: httputil.HTTPServerRequest, **target_params: Any + ) -> Optional[httputil.HTTPMessageDelegate]: if isclass(target) and issubclass(target, RequestHandler): - return self.application.get_handler_delegate(request, target, **target_params) + return self.application.get_handler_delegate( + request, target, **target_params + ) - return super(_ApplicationRouter, self).get_target_delegate(target, request, **target_params) + return super(_ApplicationRouter, self).get_target_delegate( + target, request, **target_params + ) class Application(ReversibleRouter): - """A collection of request handlers that make up a web application. + r"""A collection of request handlers that make up a web application. Instances of this class are callable and can be passed directly to HTTPServer to serve the application:: @@ -1951,7 +2014,7 @@ class Application(ReversibleRouter): Applications that do not use TLS may be vulnerable to :ref:`DNS rebinding ` attacks. This attack is especially - relevant to applications that only listen on ``127.0.0.1` or + relevant to applications that only listen on ``127.0.0.1`` or other private networks. Appropriate host patterns must be used (instead of the default of ``r'.*'``) to prevent this risk. The ``default_host`` argument must not be used in applications that @@ -1969,54 +2032,64 @@ class Application(ReversibleRouter): Integration with the new `tornado.routing` module. """ - def __init__(self, handlers=None, default_host=None, transforms=None, - **settings): + + def __init__( + self, + handlers: _RuleList = None, + default_host: str = None, + transforms: List[Type["OutputTransform"]] = None, + **settings: Any + ) -> None: if transforms is None: - self.transforms = [] + self.transforms = [] # type: List[Type[OutputTransform]] if settings.get("compress_response") or settings.get("gzip"): self.transforms.append(GZipContentEncoding) else: self.transforms = transforms self.default_host = default_host self.settings = settings - self.ui_modules = {'linkify': _linkify, - 'xsrf_form_html': _xsrf_form_html, - 'Template': TemplateModule, - } - self.ui_methods = {} + self.ui_modules = { + "linkify": _linkify, + "xsrf_form_html": _xsrf_form_html, + "Template": TemplateModule, + } + self.ui_methods = {} # type: Dict[str, Callable[..., str]] self._load_ui_modules(settings.get("ui_modules", {})) self._load_ui_methods(settings.get("ui_methods", {})) if self.settings.get("static_path"): path = self.settings["static_path"] handlers = list(handlers or []) - static_url_prefix = settings.get("static_url_prefix", - "/static/") - static_handler_class = settings.get("static_handler_class", - StaticFileHandler) + static_url_prefix = settings.get("static_url_prefix", "/static/") + static_handler_class = settings.get( + "static_handler_class", StaticFileHandler + ) static_handler_args = settings.get("static_handler_args", {}) - static_handler_args['path'] = path - for pattern in [re.escape(static_url_prefix) + r"(.*)", - r"/(favicon\.ico)", r"/(robots\.txt)"]: - handlers.insert(0, (pattern, static_handler_class, - static_handler_args)) + static_handler_args["path"] = path + for pattern in [ + re.escape(static_url_prefix) + r"(.*)", + r"/(favicon\.ico)", + r"/(robots\.txt)", + ]: + handlers.insert(0, (pattern, static_handler_class, static_handler_args)) - if self.settings.get('debug'): - self.settings.setdefault('autoreload', True) - self.settings.setdefault('compiled_template_cache', False) - self.settings.setdefault('static_hash_cache', False) - self.settings.setdefault('serve_traceback', True) + if self.settings.get("debug"): + self.settings.setdefault("autoreload", True) + self.settings.setdefault("compiled_template_cache", False) + self.settings.setdefault("static_hash_cache", False) + self.settings.setdefault("serve_traceback", True) self.wildcard_router = _ApplicationRouter(self, handlers) - self.default_router = _ApplicationRouter(self, [ - Rule(AnyMatches(), self.wildcard_router) - ]) + self.default_router = _ApplicationRouter( + self, [Rule(AnyMatches(), self.wildcard_router)] + ) # Automatically reload modified modules - if self.settings.get('autoreload'): + if self.settings.get("autoreload"): from tornado import autoreload + autoreload.start() - def listen(self, port, address="", **kwargs): + def listen(self, port: int, address: str = "", **kwargs: Any) -> HTTPServer: """Starts an HTTP server for this application on the given port. This is a convenience alias for creating an `.HTTPServer` @@ -2035,14 +2108,11 @@ class Application(ReversibleRouter): .. versionchanged:: 4.3 Now returns the `.HTTPServer` object. """ - # import is here rather than top level because HTTPServer - # is not importable on appengine - from tornado.httpserver import HTTPServer server = HTTPServer(self, **kwargs) server.listen(port, address) return server - def add_handlers(self, host_pattern, host_handlers): + def add_handlers(self, host_pattern: str, host_handlers: _RuleList) -> None: """Appends the given handlers to our handler list. Host patterns are processed sequentially in the order they were @@ -2054,31 +2124,31 @@ class Application(ReversibleRouter): self.default_router.rules.insert(-1, rule) if self.default_host is not None: - self.wildcard_router.add_rules([( - DefaultHostMatches(self, host_matcher.host_pattern), - host_handlers - )]) + self.wildcard_router.add_rules( + [(DefaultHostMatches(self, host_matcher.host_pattern), host_handlers)] + ) - def add_transform(self, transform_class): + def add_transform(self, transform_class: Type["OutputTransform"]) -> None: self.transforms.append(transform_class) - def _load_ui_methods(self, methods): + def _load_ui_methods(self, methods: Any) -> None: if isinstance(methods, types.ModuleType): - self._load_ui_methods(dict((n, getattr(methods, n)) - for n in dir(methods))) + self._load_ui_methods(dict((n, getattr(methods, n)) for n in dir(methods))) elif isinstance(methods, list): for m in methods: self._load_ui_methods(m) else: for name, fn in methods.items(): - if not name.startswith("_") and hasattr(fn, "__call__") \ - and name[0].lower() == name[0]: + if ( + not name.startswith("_") + and hasattr(fn, "__call__") + and name[0].lower() == name[0] + ): self.ui_methods[name] = fn - def _load_ui_modules(self, modules): + def _load_ui_modules(self, modules: Any) -> None: if isinstance(modules, types.ModuleType): - self._load_ui_modules(dict((n, getattr(modules, n)) - for n in dir(modules))) + self._load_ui_modules(dict((n, getattr(modules, n)) for n in dir(modules))) elif isinstance(modules, list): for m in modules: self._load_ui_modules(m) @@ -2091,27 +2161,37 @@ class Application(ReversibleRouter): except TypeError: pass - def __call__(self, request): + def __call__( + self, request: httputil.HTTPServerRequest + ) -> Optional[Awaitable[None]]: # Legacy HTTPServer interface dispatcher = self.find_handler(request) return dispatcher.execute() - def find_handler(self, request, **kwargs): + def find_handler( + self, request: httputil.HTTPServerRequest, **kwargs: Any + ) -> "_HandlerDelegate": route = self.default_router.find_handler(request) if route is not None: - return route + return cast("_HandlerDelegate", route) - if self.settings.get('default_handler_class'): + if self.settings.get("default_handler_class"): return self.get_handler_delegate( request, - self.settings['default_handler_class'], - self.settings.get('default_handler_args', {})) + self.settings["default_handler_class"], + self.settings.get("default_handler_args", {}), + ) - return self.get_handler_delegate( - request, ErrorHandler, {'status_code': 404}) + return self.get_handler_delegate(request, ErrorHandler, {"status_code": 404}) - def get_handler_delegate(self, request, target_class, target_kwargs=None, - path_args=None, path_kwargs=None): + def get_handler_delegate( + self, + request: httputil.HTTPServerRequest, + target_class: Type[RequestHandler], + target_kwargs: Dict[str, Any] = None, + path_args: List[bytes] = None, + path_kwargs: Dict[str, bytes] = None, + ) -> "_HandlerDelegate": """Returns `~.httputil.HTTPMessageDelegate` that can serve a request for application and `RequestHandler` subclass. @@ -2123,9 +2203,10 @@ class Application(ReversibleRouter): :arg dict path_kwargs: keyword arguments for ``target_class`` HTTP method. """ return _HandlerDelegate( - self, request, target_class, target_kwargs, path_args, path_kwargs) + self, request, target_class, target_kwargs, path_args, path_kwargs + ) - def reverse_url(self, name, *args): + def reverse_url(self, name: str, *args: Any) -> str: """Returns a URL path for handler named ``name`` The handler must be added to the application as a named `URLSpec`. @@ -2140,7 +2221,7 @@ class Application(ReversibleRouter): raise KeyError("%s not found in named urls" % name) - def log_request(self, handler): + def log_request(self, handler: RequestHandler) -> None: """Writes a completed HTTP request to the logs. By default writes to the python root logger. To change @@ -2158,13 +2239,24 @@ class Application(ReversibleRouter): else: log_method = access_log.error request_time = 1000.0 * handler.request.request_time() - log_method("%d %s %.2fms", handler.get_status(), - handler._request_summary(), request_time) + log_method( + "%d %s %.2fms", + handler.get_status(), + handler._request_summary(), + request_time, + ) class _HandlerDelegate(httputil.HTTPMessageDelegate): - def __init__(self, application, request, handler_class, handler_kwargs, - path_args, path_kwargs): + def __init__( + self, + application: Application, + request: httputil.HTTPServerRequest, + handler_class: Type[RequestHandler], + handler_kwargs: Optional[Dict[str, Any]], + path_args: Optional[List[bytes]], + path_kwargs: Optional[Dict[str, bytes]], + ) -> None: self.application = application self.connection = request.connection self.request = request @@ -2172,35 +2264,41 @@ class _HandlerDelegate(httputil.HTTPMessageDelegate): self.handler_kwargs = handler_kwargs or {} self.path_args = path_args or [] self.path_kwargs = path_kwargs or {} - self.chunks = [] + self.chunks = [] # type: List[bytes] self.stream_request_body = _has_stream_request_body(self.handler_class) - def headers_received(self, start_line, headers): + def headers_received( + self, + start_line: Union[httputil.RequestStartLine, httputil.ResponseStartLine], + headers: httputil.HTTPHeaders, + ) -> Optional[Awaitable[None]]: if self.stream_request_body: - self.request.body = Future() + self.request._body_future = Future() return self.execute() + return None - def data_received(self, data): + def data_received(self, data: bytes) -> Optional[Awaitable[None]]: if self.stream_request_body: return self.handler.data_received(data) else: self.chunks.append(data) + return None - def finish(self): + def finish(self) -> None: if self.stream_request_body: - future_set_result_unless_cancelled(self.request.body, None) + future_set_result_unless_cancelled(self.request._body_future, None) else: - self.request.body = b''.join(self.chunks) + self.request.body = b"".join(self.chunks) self.request._parse_body() self.execute() - def on_connection_close(self): + def on_connection_close(self) -> None: if self.stream_request_body: self.handler.on_connection_close() else: - self.chunks = None + self.chunks = None # type: ignore - def execute(self): + def execute(self) -> Optional[Awaitable[None]]: # If template cache is disabled (usually in the debug mode), # re-compile templates and reload static files on every # request so you don't need to restart to see changes @@ -2208,11 +2306,12 @@ class _HandlerDelegate(httputil.HTTPMessageDelegate): with RequestHandler._template_loader_lock: for loader in RequestHandler._template_loaders.values(): loader.reset() - if not self.application.settings.get('static_hash_cache', True): + if not self.application.settings.get("static_hash_cache", True): StaticFileHandler.reset() - self.handler = self.handler_class(self.application, self.request, - **self.handler_kwargs) + self.handler = self.handler_class( + self.application, self.request, **self.handler_kwargs + ) transforms = [t(self.request) for t in self.application.transforms] if self.stream_request_body: @@ -2224,8 +2323,10 @@ class _HandlerDelegate(httputil.HTTPMessageDelegate): # except handler, and we cannot easily access the IOLoop here to # call add_future (because of the requirement to remain compatible # with WSGI) - self.handler._execute(transforms, *self.path_args, - **self.path_kwargs) + fut = gen.convert_yielded( + self.handler._execute(transforms, *self.path_args, **self.path_kwargs) + ) + fut.add_done_callback(lambda f: f.result()) # If we are streaming the request body, then execute() is finished # when the handler has prepared to receive the body. If not, # it doesn't matter when execute() finishes (so we return None) @@ -2254,18 +2355,22 @@ class HTTPError(Exception): determined automatically from ``status_code``, but can be used to use a non-standard numeric code. """ - def __init__(self, status_code=500, log_message=None, *args, **kwargs): + + def __init__( + self, status_code: int = 500, log_message: str = None, *args: Any, **kwargs: Any + ) -> None: self.status_code = status_code self.log_message = log_message self.args = args - self.reason = kwargs.get('reason', None) + self.reason = kwargs.get("reason", None) if log_message and not args: - self.log_message = log_message.replace('%', '%%') + self.log_message = log_message.replace("%", "%%") - def __str__(self): + def __str__(self) -> str: message = "HTTP %d: %s" % ( self.status_code, - self.reason or httputil.responses.get(self.status_code, 'Unknown')) + self.reason or httputil.responses.get(self.status_code, "Unknown"), + ) if self.log_message: return message + " (" + (self.log_message % self.args) + ")" else: @@ -2296,6 +2401,7 @@ class Finish(Exception): Arguments passed to ``Finish()`` will be passed on to `RequestHandler.finish`. """ + pass @@ -2307,21 +2413,24 @@ class MissingArgumentError(HTTPError): .. versionadded:: 3.1 """ - def __init__(self, arg_name): + + def __init__(self, arg_name: str) -> None: super(MissingArgumentError, self).__init__( - 400, 'Missing argument %s' % arg_name) + 400, "Missing argument %s" % arg_name + ) self.arg_name = arg_name class ErrorHandler(RequestHandler): """Generates an error response with ``status_code`` for all requests.""" - def initialize(self, status_code): + + def initialize(self, status_code: int) -> None: self.set_status(status_code) - def prepare(self): + def prepare(self) -> None: raise HTTPError(self._status_code) - def check_xsrf_cookie(self): + def check_xsrf_cookie(self) -> None: # POSTs to an ErrorHandler don't actually have side effects, # so we don't need to check the xsrf token. This allows POSTs # to the wrong url to return a 404 instead of 403. @@ -2360,15 +2469,19 @@ class RedirectHandler(RequestHandler): If any query arguments are present, they will be copied to the destination URL. """ - def initialize(self, url, permanent=True): + + def initialize(self, url: str, permanent: bool = True) -> None: self._url = url self._permanent = permanent - def get(self, *args): + def get(self, *args: Any) -> None: to_url = self._url.format(*args) if self.request.query_arguments: + # TODO: figure out typing for the next line. to_url = httputil.url_concat( - to_url, list(httputil.qs_to_qsl(self.request.query_arguments))) + to_url, + list(httputil.qs_to_qsl(self.request.query_arguments)), # type: ignore + ) self.redirect(to_url, permanent=self._permanent) @@ -2438,31 +2551,30 @@ class StaticFileHandler(RequestHandler): .. versionchanged:: 3.1 Many of the methods for subclasses were added in Tornado 3.1. """ + CACHE_MAX_AGE = 86400 * 365 * 10 # 10 years - _static_hashes = {} # type: typing.Dict + _static_hashes = {} # type: Dict[str, Optional[str]] _lock = threading.Lock() # protects _static_hashes - def initialize(self, path, default_filename=None): + def initialize(self, path: str, default_filename: str = None) -> None: self.root = path self.default_filename = default_filename @classmethod - def reset(cls): + def reset(cls) -> None: with cls._lock: cls._static_hashes = {} - def head(self, path): + def head(self, path: str) -> Awaitable[None]: return self.get(path, include_body=False) - @gen.coroutine - def get(self, path, include_body=True): + async def get(self, path: str, include_body: bool = True) -> None: # Set up our path instance variables. self.path = self.parse_url_path(path) del path # make sure we don't refer to path instead of self.path again absolute_path = self.get_absolute_path(self.root, self.path) - self.absolute_path = self.validate_absolute_path( - self.root, absolute_path) + self.absolute_path = self.validate_absolute_path(self.root, absolute_path) if self.absolute_path is None: return @@ -2483,16 +2595,24 @@ class StaticFileHandler(RequestHandler): size = self.get_content_size() if request_range: start, end = request_range - if (start is not None and start >= size) or end == 0: - # As per RFC 2616 14.35.1, a range is not satisfiable only: if - # the first requested byte is equal to or greater than the - # content, or when a suffix with length 0 is specified - self.set_status(416) # Range Not Satisfiable - self.set_header("Content-Type", "text/plain") - self.set_header("Content-Range", "bytes */%s" % (size, )) - return if start is not None and start < 0: start += size + if start < 0: + start = 0 + if ( + start is not None + and (start >= size or (end is not None and start >= end)) + ) or end == 0: + # As per RFC 2616 14.35.1, a range is not satisfiable only: if + # the first requested byte is equal to or greater than the + # content, or when a suffix with length 0 is specified. + # https://tools.ietf.org/html/rfc7233#section-2.1 + # A byte-range-spec is invalid if the last-byte-pos value is present + # and less than the first-byte-pos. + self.set_status(416) # Range Not Satisfiable + self.set_header("Content-Type", "text/plain") + self.set_header("Content-Range", "bytes */%s" % (size,)) + return if end is not None and end > size: # Clients sometimes blindly use a large range to limit their # download size; cap the endpoint at the actual file size. @@ -2503,8 +2623,9 @@ class StaticFileHandler(RequestHandler): # ``Range: bytes=0-``. if size != (end or size) - (start or 0): self.set_status(206) # Partial Content - self.set_header("Content-Range", - httputil._get_content_range(start, end, size)) + self.set_header( + "Content-Range", httputil._get_content_range(start, end, size) + ) else: start = end = None @@ -2525,13 +2646,13 @@ class StaticFileHandler(RequestHandler): for chunk in content: try: self.write(chunk) - yield self.flush() + await self.flush() except iostream.StreamClosedError: return else: assert self.request.method == "HEAD" - def compute_etag(self): + def compute_etag(self) -> Optional[str]: """Sets the ``Etag`` header based on static url version. This allows efficient ``If-None-Match`` checks against cached @@ -2540,12 +2661,13 @@ class StaticFileHandler(RequestHandler): .. versionadded:: 3.1 """ + assert self.absolute_path is not None version_hash = self._get_cached_version(self.absolute_path) if not version_hash: return None - return '"%s"' % (version_hash, ) + return '"%s"' % (version_hash,) - def set_headers(self): + def set_headers(self) -> None: """Sets the content and caching headers on the response. .. versionadded:: 3.1 @@ -2560,22 +2682,23 @@ class StaticFileHandler(RequestHandler): if content_type: self.set_header("Content-Type", content_type) - cache_time = self.get_cache_time(self.path, self.modified, - content_type) + cache_time = self.get_cache_time(self.path, self.modified, content_type) if cache_time > 0: - self.set_header("Expires", datetime.datetime.utcnow() + - datetime.timedelta(seconds=cache_time)) + self.set_header( + "Expires", + datetime.datetime.utcnow() + datetime.timedelta(seconds=cache_time), + ) self.set_header("Cache-Control", "max-age=" + str(cache_time)) self.set_extra_headers(self.path) - def should_return_304(self): + def should_return_304(self) -> bool: """Returns True if the headers indicate that we should return 304. .. versionadded:: 3.1 """ # If client sent If-None-Match, use it, ignore If-Modified-Since - if self.request.headers.get('If-None-Match'): + if self.request.headers.get("If-None-Match"): return self.check_etag_header() # Check the If-Modified-Since, and don't send the result if the @@ -2585,13 +2708,14 @@ class StaticFileHandler(RequestHandler): date_tuple = email.utils.parsedate(ims_value) if date_tuple is not None: if_since = datetime.datetime(*date_tuple[:6]) + assert self.modified is not None if if_since >= self.modified: return True return False @classmethod - def get_absolute_path(cls, root, path): + def get_absolute_path(cls, root: str, path: str) -> str: """Returns the absolute location of ``path`` relative to ``root``. ``root`` is the path configured for this `StaticFileHandler` @@ -2607,7 +2731,7 @@ class StaticFileHandler(RequestHandler): abspath = os.path.abspath(os.path.join(root, path)) return abspath - def validate_absolute_path(self, root, absolute_path): + def validate_absolute_path(self, root: str, absolute_path: str) -> Optional[str]: """Validate and return the absolute path. ``root`` is the configured path for the `StaticFileHandler`, @@ -2642,16 +2766,14 @@ class StaticFileHandler(RequestHandler): # The trailing slash also needs to be temporarily added back # the requested path so a request to root/ will match. if not (absolute_path + os.path.sep).startswith(root): - raise HTTPError(403, "%s is not in root static directory", - self.path) - if (os.path.isdir(absolute_path) and - self.default_filename is not None): + raise HTTPError(403, "%s is not in root static directory", self.path) + if os.path.isdir(absolute_path) and self.default_filename is not None: # need to look at the request.path here for when path is empty # but there is some prefix to the path that was already # trimmed by the routing if not self.request.path.endswith("/"): self.redirect(self.request.path + "/", permanent=True) - return + return None absolute_path = os.path.join(absolute_path, self.default_filename) if not os.path.exists(absolute_path): raise HTTPError(404) @@ -2660,7 +2782,9 @@ class StaticFileHandler(RequestHandler): return absolute_path @classmethod - def get_content(cls, abspath, start=None, end=None): + def get_content( + cls, abspath: str, start: int = None, end: int = None + ) -> Generator[bytes, None, None]: """Retrieve the content of the requested resource which is located at the given absolute path. @@ -2679,7 +2803,7 @@ class StaticFileHandler(RequestHandler): if start is not None: file.seek(start) if end is not None: - remaining = end - (start or 0) + remaining = end - (start or 0) # type: Optional[int] else: remaining = None while True: @@ -2697,7 +2821,7 @@ class StaticFileHandler(RequestHandler): return @classmethod - def get_content_version(cls, abspath): + def get_content_version(cls, abspath: str) -> str: """Returns a version string for the resource at the given path. This class method may be overridden by subclasses. The @@ -2714,12 +2838,13 @@ class StaticFileHandler(RequestHandler): hasher.update(chunk) return hasher.hexdigest() - def _stat(self): - if not hasattr(self, '_stat_result'): + def _stat(self) -> os.stat_result: + assert self.absolute_path is not None + if not hasattr(self, "_stat_result"): self._stat_result = os.stat(self.absolute_path) return self._stat_result - def get_content_size(self): + def get_content_size(self) -> int: """Retrieve the total size of the resource at the given path. This method may be overridden by subclasses. @@ -2731,9 +2856,9 @@ class StaticFileHandler(RequestHandler): partial results are requested. """ stat_result = self._stat() - return stat_result[stat.ST_SIZE] + return stat_result.st_size - def get_modified_time(self): + def get_modified_time(self) -> Optional[datetime.datetime]: """Returns the time that ``self.absolute_path`` was last modified. May be overridden in subclasses. Should return a `~datetime.datetime` @@ -2742,15 +2867,23 @@ class StaticFileHandler(RequestHandler): .. versionadded:: 3.1 """ stat_result = self._stat() - modified = datetime.datetime.utcfromtimestamp( - stat_result[stat.ST_MTIME]) + # NOTE: Historically, this used stat_result[stat.ST_MTIME], + # which truncates the fractional portion of the timestamp. It + # was changed from that form to stat_result.st_mtime to + # satisfy mypy (which disallows the bracket operator), but the + # latter form returns a float instead of an int. For + # consistency with the past (and because we have a unit test + # that relies on this), we truncate the float here, although + # I'm not sure that's the right thing to do. + modified = datetime.datetime.utcfromtimestamp(int(stat_result.st_mtime)) return modified - def get_content_type(self): + def get_content_type(self) -> str: """Returns the ``Content-Type`` header to be used for this request. .. versionadded:: 3.1 """ + assert self.absolute_path is not None mime_type, encoding = mimetypes.guess_type(self.absolute_path) # per RFC 6713, use the appropriate type for a gzip compressed file if encoding == "gzip": @@ -2766,11 +2899,13 @@ class StaticFileHandler(RequestHandler): else: return "application/octet-stream" - def set_extra_headers(self, path): + def set_extra_headers(self, path: str) -> None: """For subclass to add extra headers to the response""" pass - def get_cache_time(self, path, modified, mime_type): + def get_cache_time( + self, path: str, modified: Optional[datetime.datetime], mime_type: str + ) -> int: """Override to customize cache control behavior. Return a positive number of seconds to make the result @@ -2784,7 +2919,9 @@ class StaticFileHandler(RequestHandler): return self.CACHE_MAX_AGE if "v" in self.request.arguments else 0 @classmethod - def make_static_url(cls, settings, path, include_version=True): + def make_static_url( + cls, settings: Dict[str, Any], path: str, include_version: bool = True + ) -> str: """Constructs a versioned url for the given path. This method may be overridden in subclasses (but note that it @@ -2803,7 +2940,7 @@ class StaticFileHandler(RequestHandler): file corresponding to the given ``path``. """ - url = settings.get('static_url_prefix', '/static/') + path + url = settings.get("static_url_prefix", "/static/") + path if not include_version: return url @@ -2811,9 +2948,9 @@ class StaticFileHandler(RequestHandler): if not version_hash: return url - return '%s?v=%s' % (url, version_hash) + return "%s?v=%s" % (url, version_hash) - def parse_url_path(self, url_path): + def parse_url_path(self, url_path: str) -> str: """Converts a static URL path into a filesystem path. ``url_path`` is the path component of the URL with @@ -2827,7 +2964,7 @@ class StaticFileHandler(RequestHandler): return url_path @classmethod - def get_version(cls, settings, path): + def get_version(cls, settings: Dict[str, Any], path: str) -> Optional[str]: """Generate the version string to be used in static URLs. ``settings`` is the `Application.settings` dictionary and ``path`` @@ -2840,11 +2977,11 @@ class StaticFileHandler(RequestHandler): `get_content_version` is now preferred as it allows the base class to handle caching of the result. """ - abs_path = cls.get_absolute_path(settings['static_path'], path) + abs_path = cls.get_absolute_path(settings["static_path"], path) return cls._get_cached_version(abs_path) @classmethod - def _get_cached_version(cls, abs_path): + def _get_cached_version(cls, abs_path: str) -> Optional[str]: with cls._lock: hashes = cls._static_hashes if abs_path not in hashes: @@ -2875,10 +3012,13 @@ class FallbackHandler(RequestHandler): (r".*", FallbackHandler, dict(fallback=wsgi_app), ]) """ - def initialize(self, fallback): + + def initialize( + self, fallback: Callable[[httputil.HTTPServerRequest], None] + ) -> None: self.fallback = fallback - def prepare(self): + def prepare(self) -> None: self.fallback(self.request) self._finished = True self.on_finish() @@ -2891,14 +3031,20 @@ class OutputTransform(object): or interact with them directly; the framework chooses which transforms (if any) to apply. """ - def __init__(self, request): + + def __init__(self, request: httputil.HTTPServerRequest) -> None: pass - def transform_first_chunk(self, status_code, headers, chunk, finishing): - # type: (int, httputil.HTTPHeaders, bytes, bool) -> typing.Tuple[int, httputil.HTTPHeaders, bytes] # noqa: E501 + def transform_first_chunk( + self, + status_code: int, + headers: httputil.HTTPHeaders, + chunk: bytes, + finishing: bool, + ) -> Tuple[int, httputil.HTTPHeaders, bytes]: return status_code, headers, chunk - def transform_chunk(self, chunk, finishing): + def transform_chunk(self, chunk: bytes, finishing: bool) -> bytes: return chunk @@ -2912,12 +3058,20 @@ class GZipContentEncoding(OutputTransform): of just a whitelist. (the whitelist is still used for certain non-text mime types). """ + # Whitelist of compressible mime types (in addition to any types # beginning with "text/"). - CONTENT_TYPES = set(["application/javascript", "application/x-javascript", - "application/xml", "application/atom+xml", - "application/json", "application/xhtml+xml", - "image/svg+xml"]) + CONTENT_TYPES = set( + [ + "application/javascript", + "application/x-javascript", + "application/xml", + "application/atom+xml", + "application/json", + "application/xhtml+xml", + "image/svg+xml", + ] + ) # Python's GzipFile defaults to level 9, while most other gzip # tools (including gzip itself) default to 6, which is probably a # better CPU/size tradeoff. @@ -2929,29 +3083,37 @@ class GZipContentEncoding(OutputTransform): # regardless of size. MIN_LENGTH = 1024 - def __init__(self, request): + def __init__(self, request: httputil.HTTPServerRequest) -> None: self._gzipping = "gzip" in request.headers.get("Accept-Encoding", "") - def _compressible_type(self, ctype): - return ctype.startswith('text/') or ctype in self.CONTENT_TYPES + def _compressible_type(self, ctype: str) -> bool: + return ctype.startswith("text/") or ctype in self.CONTENT_TYPES - def transform_first_chunk(self, status_code, headers, chunk, finishing): - # type: (int, httputil.HTTPHeaders, bytes, bool) -> typing.Tuple[int, httputil.HTTPHeaders, bytes] # noqa: E501 + def transform_first_chunk( + self, + status_code: int, + headers: httputil.HTTPHeaders, + chunk: bytes, + finishing: bool, + ) -> Tuple[int, httputil.HTTPHeaders, bytes]: # TODO: can/should this type be inherited from the superclass? - if 'Vary' in headers: - headers['Vary'] += ', Accept-Encoding' + if "Vary" in headers: + headers["Vary"] += ", Accept-Encoding" else: - headers['Vary'] = 'Accept-Encoding' + headers["Vary"] = "Accept-Encoding" if self._gzipping: ctype = _unicode(headers.get("Content-Type", "")).split(";")[0] - self._gzipping = self._compressible_type(ctype) and \ - (not finishing or len(chunk) >= self.MIN_LENGTH) and \ - ("Content-Encoding" not in headers) + self._gzipping = ( + self._compressible_type(ctype) + and (not finishing or len(chunk) >= self.MIN_LENGTH) + and ("Content-Encoding" not in headers) + ) if self._gzipping: headers["Content-Encoding"] = "gzip" self._gzip_value = BytesIO() - self._gzip_file = gzip.GzipFile(mode="w", fileobj=self._gzip_value, - compresslevel=self.GZIP_LEVEL) + self._gzip_file = gzip.GzipFile( + mode="w", fileobj=self._gzip_value, compresslevel=self.GZIP_LEVEL + ) chunk = self.transform_chunk(chunk, finishing) if "Content-Length" in headers: # The original content length is no longer correct. @@ -2964,7 +3126,7 @@ class GZipContentEncoding(OutputTransform): del headers["Content-Length"] return status_code, headers, chunk - def transform_chunk(self, chunk, finishing): + def transform_chunk(self, chunk: bytes, finishing: bool) -> bytes: if self._gzipping: self._gzip_file.write(chunk) if finishing: @@ -2977,7 +3139,9 @@ class GZipContentEncoding(OutputTransform): return chunk -def authenticated(method): +def authenticated( + method: Callable[..., Optional[Awaitable[None]]] +) -> Callable[..., Optional[Awaitable[None]]]: """Decorate methods with this to require that the user be logged in. If the user is not logged in, they will be redirected to the configured @@ -2988,22 +3152,27 @@ def authenticated(method): will add a `next` parameter so the login page knows where to send you once you're logged in. """ + @functools.wraps(method) - def wrapper(self, *args, **kwargs): + def wrapper( # type: ignore + self: RequestHandler, *args, **kwargs + ) -> Optional[Awaitable[None]]: if not self.current_user: if self.request.method in ("GET", "HEAD"): url = self.get_login_url() if "?" not in url: - if urlparse.urlsplit(url).scheme: + if urllib.parse.urlsplit(url).scheme: # if login url is absolute, make next absolute too next_url = self.request.full_url() else: + assert self.request.uri is not None next_url = self.request.uri url += "?" + urlencode(dict(next=next_url)) self.redirect(url) - return + return None raise HTTPError(403) return method(self, *args, **kwargs) + return wrapper @@ -3016,26 +3185,27 @@ class UIModule(object): Subclasses of UIModule must override the `render` method. """ - def __init__(self, handler): + + def __init__(self, handler: RequestHandler) -> None: self.handler = handler self.request = handler.request self.ui = handler.ui self.locale = handler.locale @property - def current_user(self): + def current_user(self) -> Any: return self.handler.current_user - def render(self, *args, **kwargs): + def render(self, *args: Any, **kwargs: Any) -> str: """Override in subclasses to return this module's output.""" raise NotImplementedError() - def embedded_javascript(self): + def embedded_javascript(self) -> Optional[str]: """Override to return a JavaScript string to be embedded in the page.""" return None - def javascript_files(self): + def javascript_files(self) -> Optional[Iterable[str]]: """Override to return a list of JavaScript files needed by this module. If the return values are relative paths, they will be passed to @@ -3043,12 +3213,12 @@ class UIModule(object): """ return None - def embedded_css(self): + def embedded_css(self) -> Optional[str]: """Override to return a CSS string that will be embedded in the page.""" return None - def css_files(self): + def css_files(self) -> Optional[Iterable[str]]: """Override to returns a list of CSS files required by this module. If the return values are relative paths, they will be passed to @@ -3056,30 +3226,30 @@ class UIModule(object): """ return None - def html_head(self): + def html_head(self) -> Optional[str]: """Override to return an HTML string that will be put in the element. """ return None - def html_body(self): + def html_body(self) -> Optional[str]: """Override to return an HTML string that will be put at the end of the element. """ return None - def render_string(self, path, **kwargs): + def render_string(self, path: str, **kwargs: Any) -> bytes: """Renders a template and returns it as a string.""" return self.handler.render_string(path, **kwargs) class _linkify(UIModule): - def render(self, text, **kwargs): + def render(self, text: str, **kwargs: Any) -> str: # type: ignore return escape.linkify(text, **kwargs) class _xsrf_form_html(UIModule): - def render(self): + def render(self) -> str: # type: ignore return self.handler.xsrf_form_html() @@ -3098,32 +3268,35 @@ class TemplateModule(UIModule): per instantiation of the template, so they must not depend on any arguments to the template. """ - def __init__(self, handler): + + def __init__(self, handler: RequestHandler) -> None: super(TemplateModule, self).__init__(handler) # keep resources in both a list and a dict to preserve order - self._resource_list = [] - self._resource_dict = {} + self._resource_list = [] # type: List[Dict[str, Any]] + self._resource_dict = {} # type: Dict[str, Dict[str, Any]] - def render(self, path, **kwargs): - def set_resources(**kwargs): + def render(self, path: str, **kwargs: Any) -> bytes: # type: ignore + def set_resources(**kwargs) -> str: # type: ignore if path not in self._resource_dict: self._resource_list.append(kwargs) self._resource_dict[path] = kwargs else: if self._resource_dict[path] != kwargs: - raise ValueError("set_resources called with different " - "resources for the same template") + raise ValueError( + "set_resources called with different " + "resources for the same template" + ) return "" - return self.render_string(path, set_resources=set_resources, - **kwargs) - def _get_resources(self, key): + return self.render_string(path, set_resources=set_resources, **kwargs) + + def _get_resources(self, key: str) -> Iterable[str]: return (r[key] for r in self._resource_list if key in r) - def embedded_javascript(self): + def embedded_javascript(self) -> str: return "\n".join(self._get_resources("embedded_javascript")) - def javascript_files(self): + def javascript_files(self) -> Iterable[str]: result = [] for f in self._get_resources("javascript_files"): if isinstance(f, (unicode_type, bytes)): @@ -3132,10 +3305,10 @@ class TemplateModule(UIModule): result.extend(f) return result - def embedded_css(self): + def embedded_css(self) -> str: return "\n".join(self._get_resources("embedded_css")) - def css_files(self): + def css_files(self) -> Iterable[str]: result = [] for f in self._get_resources("css_files"): if isinstance(f, (unicode_type, bytes)): @@ -3144,47 +3317,40 @@ class TemplateModule(UIModule): result.extend(f) return result - def html_head(self): + def html_head(self) -> str: return "".join(self._get_resources("html_head")) - def html_body(self): + def html_body(self) -> str: return "".join(self._get_resources("html_body")) class _UIModuleNamespace(object): """Lazy namespace which creates UIModule proxies bound to a handler.""" - def __init__(self, handler, ui_modules): + + def __init__( + self, handler: RequestHandler, ui_modules: Dict[str, Type[UIModule]] + ) -> None: self.handler = handler self.ui_modules = ui_modules - def __getitem__(self, key): + def __getitem__(self, key: str) -> Callable[..., str]: return self.handler._ui_module(key, self.ui_modules[key]) - def __getattr__(self, key): + def __getattr__(self, key: str) -> Callable[..., str]: try: return self[key] except KeyError as e: raise AttributeError(str(e)) -if hasattr(hmac, 'compare_digest'): # python 3.3 - _time_independent_equals = hmac.compare_digest -else: - def _time_independent_equals(a, b): - if len(a) != len(b): - return False - result = 0 - if isinstance(a[0], int): # python3 byte strings - for x, y in zip(a, b): - result |= x ^ y - else: # python2 - for x, y in zip(a, b): - result |= ord(x) ^ ord(y) - return result == 0 - - -def create_signed_value(secret, name, value, version=None, clock=None, - key_version=None): +def create_signed_value( + secret: _CookieSecretTypes, + name: str, + value: Union[str, bytes], + version: int = None, + clock: Callable[[], float] = None, + key_version: int = None, +) -> bytes: if version is None: version = DEFAULT_SIGNED_VALUE_VERSION if clock is None: @@ -3193,6 +3359,7 @@ def create_signed_value(secret, name, value, version=None, clock=None, timestamp = utf8(str(int(clock()))) value = base64.b64encode(utf8(value)) if version == 1: + assert not isinstance(secret, dict) signature = _create_signature_v1(secret, name, value, timestamp) value = b"|".join([value, timestamp, signature]) return value @@ -3211,19 +3378,25 @@ def create_signed_value(secret, name, value, version=None, clock=None, # - name (not encoded; assumed to be ~alphanumeric) # - value (base64-encoded) # - signature (hex-encoded; no length prefix) - def format_field(s): + def format_field(s: Union[str, bytes]) -> bytes: return utf8("%d:" % len(s)) + utf8(s) - to_sign = b"|".join([ - b"2", - format_field(str(key_version or 0)), - format_field(timestamp), - format_field(name), - format_field(value), - b'']) + + to_sign = b"|".join( + [ + b"2", + format_field(str(key_version or 0)), + format_field(timestamp), + format_field(name), + format_field(value), + b"", + ] + ) if isinstance(secret, dict): - assert key_version is not None, 'Key version must be set when sign key dict is used' - assert version >= 2, 'Version must be at least 2 for key version support' + assert ( + key_version is not None + ), "Key version must be set when sign key dict is used" + assert version >= 2, "Version must be at least 2 for key version support" secret = secret[key_version] signature = _create_signature_v2(secret, to_sign) @@ -3237,7 +3410,7 @@ def create_signed_value(secret, name, value, version=None, clock=None, _signed_value_version_re = re.compile(br"^([1-9][0-9]*)\|(.*)$") -def _get_version(value): +def _get_version(value: bytes) -> int: # Figures out what version value is. Version 1 did not include an # explicit version field and started with arbitrary base64 data, # which makes this tricky. @@ -3260,8 +3433,14 @@ def _get_version(value): return version -def decode_signed_value(secret, name, value, max_age_days=31, - clock=None, min_version=None): +def decode_signed_value( + secret: _CookieSecretTypes, + name: str, + value: Union[None, str, bytes], + max_age_days: int = 31, + clock: Callable[[], float] = None, + min_version: int = None, +) -> Optional[bytes]: if clock is None: clock = time.time if min_version is None: @@ -3277,21 +3456,26 @@ def decode_signed_value(secret, name, value, max_age_days=31, if version < min_version: return None if version == 1: - return _decode_signed_value_v1(secret, name, value, - max_age_days, clock) + assert not isinstance(secret, dict) + return _decode_signed_value_v1(secret, name, value, max_age_days, clock) elif version == 2: - return _decode_signed_value_v2(secret, name, value, - max_age_days, clock) + return _decode_signed_value_v2(secret, name, value, max_age_days, clock) else: return None -def _decode_signed_value_v1(secret, name, value, max_age_days, clock): +def _decode_signed_value_v1( + secret: Union[str, bytes], + name: str, + value: bytes, + max_age_days: int, + clock: Callable[[], float], +) -> Optional[bytes]: parts = utf8(value).split(b"|") if len(parts) != 3: return None signature = _create_signature_v1(secret, name, parts[0], parts[1]) - if not _time_independent_equals(parts[2], signature): + if not hmac.compare_digest(parts[2], signature): gen_log.warning("Invalid cookie signature %r", value) return None timestamp = int(parts[1]) @@ -3304,8 +3488,7 @@ def _decode_signed_value_v1(secret, name, value, max_age_days, clock): # digits from the payload to the timestamp without altering the # signature. For backwards compatibility, sanity-check timestamp # here instead of modifying _cookie_signature. - gen_log.warning("Cookie timestamp in future; possible tampering %r", - value) + gen_log.warning("Cookie timestamp in future; possible tampering %r", value) return None if parts[1].startswith(b"0"): gen_log.warning("Tampered cookie %r", value) @@ -3316,16 +3499,16 @@ def _decode_signed_value_v1(secret, name, value, max_age_days, clock): return None -def _decode_fields_v2(value): - def _consume_field(s): - length, _, rest = s.partition(b':') +def _decode_fields_v2(value: bytes) -> Tuple[int, bytes, bytes, bytes, bytes]: + def _consume_field(s: bytes) -> Tuple[bytes, bytes]: + length, _, rest = s.partition(b":") n = int(length) field_value = rest[:n] # In python 3, indexing bytes returns small integers; we must # use a slice to get a byte string as in python 2. - if rest[n:n + 1] != b'|': + if rest[n : n + 1] != b"|": raise ValueError("malformed v2 signed value field") - rest = rest[n + 1:] + rest = rest[n + 1 :] return field_value, rest rest = value[2:] # remove version number @@ -3336,12 +3519,20 @@ def _decode_fields_v2(value): return int(key_version), timestamp, name_field, value_field, passed_sig -def _decode_signed_value_v2(secret, name, value, max_age_days, clock): +def _decode_signed_value_v2( + secret: _CookieSecretTypes, + name: str, + value: bytes, + max_age_days: int, + clock: Callable[[], float], +) -> Optional[bytes]: try: - key_version, timestamp, name_field, value_field, passed_sig = _decode_fields_v2(value) + key_version, timestamp_bytes, name_field, value_field, passed_sig = _decode_fields_v2( + value + ) except ValueError: return None - signed_string = value[:-len(passed_sig)] + signed_string = value[: -len(passed_sig)] if isinstance(secret, dict): try: @@ -3350,11 +3541,11 @@ def _decode_signed_value_v2(secret, name, value, max_age_days, clock): return None expected_sig = _create_signature_v2(secret, signed_string) - if not _time_independent_equals(passed_sig, expected_sig): + if not hmac.compare_digest(passed_sig, expected_sig): return None if name_field != utf8(name): return None - timestamp = int(timestamp) + timestamp = int(timestamp_bytes) if timestamp < clock() - max_age_days * 86400: # The signature has expired. return None @@ -3364,7 +3555,7 @@ def _decode_signed_value_v2(secret, name, value, max_age_days, clock): return None -def get_signature_key_version(value): +def get_signature_key_version(value: Union[str, bytes]) -> Optional[int]: value = utf8(value) version = _get_version(value) if version < 2: @@ -3377,18 +3568,18 @@ def get_signature_key_version(value): return key_version -def _create_signature_v1(secret, *parts): +def _create_signature_v1(secret: Union[str, bytes], *parts: Union[str, bytes]) -> bytes: hash = hmac.new(utf8(secret), digestmod=hashlib.sha1) for part in parts: hash.update(utf8(part)) return utf8(hash.hexdigest()) -def _create_signature_v2(secret, s): +def _create_signature_v2(secret: Union[str, bytes], s: bytes) -> bytes: hash = hmac.new(utf8(secret), digestmod=hashlib.sha256) hash.update(utf8(s)) return utf8(hash.hexdigest()) -def is_absolute(path): +def is_absolute(path: str) -> bool: return any(path.startswith(x) for x in ["/", "http:", "https:"]) diff --git a/server/www/packages/packages-linux/x64/tornado/websocket.py b/server/www/packages/packages-linux/x64/tornado/websocket.py index 0b994fc..d991fee 100644 --- a/server/www/packages/packages-linux/x64/tornado/websocket.py +++ b/server/www/packages/packages-linux/x64/tornado/websocket.py @@ -16,8 +16,8 @@ the protocol (known as "draft 76") and are not compatible with this module. Removed support for the draft 76 protocol version. """ -from __future__ import absolute_import, division, print_function - +import abc +import asyncio import base64 import hashlib import os @@ -25,24 +25,79 @@ import sys import struct import tornado.escape import tornado.web +from urllib.parse import urlparse import zlib from tornado.concurrent import Future, future_set_result_unless_cancelled from tornado.escape import utf8, native_str, to_unicode from tornado import gen, httpclient, httputil from tornado.ioloop import IOLoop, PeriodicCallback -from tornado.iostream import StreamClosedError -from tornado.log import gen_log +from tornado.iostream import StreamClosedError, IOStream +from tornado.log import gen_log, app_log from tornado import simple_httpclient from tornado.queues import Queue from tornado.tcpclient import TCPClient -from tornado.util import _websocket_mask, PY3 +from tornado.util import _websocket_mask + +from typing import ( + TYPE_CHECKING, + cast, + Any, + Optional, + Dict, + Union, + List, + Awaitable, + Callable, + Tuple, + Type, +) +from types import TracebackType + +if TYPE_CHECKING: + from typing_extensions import Protocol + + # The zlib compressor types aren't actually exposed anywhere + # publicly, so declare protocols for the portions we use. + class _Compressor(Protocol): + def compress(self, data: bytes) -> bytes: + pass + + def flush(self, mode: int) -> bytes: + pass + + class _Decompressor(Protocol): + unconsumed_tail = b"" # type: bytes + + def decompress(self, data: bytes, max_length: int) -> bytes: + pass + + class _WebSocketDelegate(Protocol): + # The common base interface implemented by WebSocketHandler on + # the server side and WebSocketClientConnection on the client + # side. + def on_ws_connection_close( + self, close_code: int = None, close_reason: str = None + ) -> None: + pass + + def on_message(self, message: Union[str, bytes]) -> Optional["Awaitable[None]"]: + pass + + def on_ping(self, data: bytes) -> None: + pass + + def on_pong(self, data: bytes) -> None: + pass + + def log_exception( + self, + typ: Optional[Type[BaseException]], + value: Optional[BaseException], + tb: Optional[TracebackType], + ) -> None: + pass -if PY3: - from urllib.parse import urlparse # py2 - xrange = range -else: - from urlparse import urlparse # py3 _default_max_message_size = 10 * 1024 * 1024 @@ -56,6 +111,7 @@ class WebSocketClosedError(WebSocketError): .. versionadded:: 3.2 """ + pass @@ -63,6 +119,20 @@ class _DecompressTooLargeError(Exception): pass +class _WebSocketParams(object): + def __init__( + self, + ping_interval: float = None, + ping_timeout: float = None, + max_message_size: int = _default_max_message_size, + compression_options: Dict[str, Any] = None, + ) -> None: + self.ping_interval = ping_interval + self.ping_timeout = ping_timeout + self.max_message_size = max_message_size + self.compression_options = compression_options + + class WebSocketHandler(tornado.web.RequestHandler): """Subclass this class to create a basic WebSocket handler. @@ -144,22 +214,28 @@ class WebSocketHandler(tornado.web.RequestHandler): Added ``websocket_ping_interval``, ``websocket_ping_timeout``, and ``websocket_max_message_size``. """ - def __init__(self, application, request, **kwargs): + + def __init__( + self, + application: tornado.web.Application, + request: httputil.HTTPServerRequest, + **kwargs: Any + ) -> None: super(WebSocketHandler, self).__init__(application, request, **kwargs) - self.ws_connection = None - self.close_code = None - self.close_reason = None - self.stream = None + self.ws_connection = None # type: Optional[WebSocketProtocol] + self.close_code = None # type: Optional[int] + self.close_reason = None # type: Optional[str] + self.stream = None # type: Optional[IOStream] self._on_close_called = False - def get(self, *args, **kwargs): + async def get(self, *args: Any, **kwargs: Any) -> None: self.open_args = args self.open_kwargs = kwargs # Upgrade header should be present and should be equal to WebSocket - if self.request.headers.get("Upgrade", "").lower() != 'websocket': + if self.request.headers.get("Upgrade", "").lower() != "websocket": self.set_status(400) - log_msg = "Can \"Upgrade\" only to \"WebSocket\"." + log_msg = 'Can "Upgrade" only to "WebSocket".' self.finish(log_msg) gen_log.debug(log_msg) return @@ -168,11 +244,12 @@ class WebSocketHandler(tornado.web.RequestHandler): # Some proxy servers/load balancers # might mess with it. headers = self.request.headers - connection = map(lambda s: s.strip().lower(), - headers.get("Connection", "").split(",")) - if 'upgrade' not in connection: + connection = map( + lambda s: s.strip().lower(), headers.get("Connection", "").split(",") + ) + if "upgrade" not in connection: self.set_status(400) - log_msg = "\"Connection\" must be \"Upgrade\"." + log_msg = '"Connection" must be "Upgrade".' self.finish(log_msg) gen_log.debug(log_msg) return @@ -198,32 +275,31 @@ class WebSocketHandler(tornado.web.RequestHandler): self.ws_connection = self.get_websocket_protocol() if self.ws_connection: - self.ws_connection.accept_connection() + await self.ws_connection.accept_connection(self) else: self.set_status(426, "Upgrade Required") self.set_header("Sec-WebSocket-Version", "7, 8, 13") - self.finish() stream = None @property - def ping_interval(self): + def ping_interval(self) -> Optional[float]: """The interval for websocket keep-alive pings. Set websocket_ping_interval = 0 to disable pings. """ - return self.settings.get('websocket_ping_interval', None) + return self.settings.get("websocket_ping_interval", None) @property - def ping_timeout(self): + def ping_timeout(self) -> Optional[float]: """If no ping is received in this many seconds, close the websocket connection (VPNs, etc. can fail to cleanly close ws connections). Default is max of 3 pings or 30 seconds. """ - return self.settings.get('websocket_ping_timeout', None) + return self.settings.get("websocket_ping_timeout", None) @property - def max_message_size(self): + def max_message_size(self) -> int: """Maximum allowed message size. If the remote peer sends a message larger than this, the connection @@ -231,9 +307,13 @@ class WebSocketHandler(tornado.web.RequestHandler): Default is 10MiB. """ - return self.settings.get('websocket_max_message_size', _default_max_message_size) + return self.settings.get( + "websocket_max_message_size", _default_max_message_size + ) - def write_message(self, message, binary=False): + def write_message( + self, message: Union[bytes, str, Dict[str, Any]], binary: bool = False + ) -> "Future[None]": """Sends the given message to the client of this Web Socket. The message may be either a string or a dict (which will be @@ -255,13 +335,13 @@ class WebSocketHandler(tornado.web.RequestHandler): Consistently raises `WebSocketClosedError`. Previously could sometimes raise `.StreamClosedError`. """ - if self.ws_connection is None: + if self.ws_connection is None or self.ws_connection.is_closing(): raise WebSocketClosedError() if isinstance(message, dict): message = tornado.escape.json_encode(message) return self.ws_connection.write_message(message, binary=binary) - def select_subprotocol(self, subprotocols): + def select_subprotocol(self, subprotocols: List[str]) -> Optional[str]: """Override to implement subprotocol negotiation. ``subprotocols`` is a list of strings identifying the @@ -287,14 +367,15 @@ class WebSocketHandler(tornado.web.RequestHandler): return None @property - def selected_subprotocol(self): + def selected_subprotocol(self) -> Optional[str]: """The subprotocol returned by `select_subprotocol`. .. versionadded:: 5.1 """ + assert self.ws_connection is not None return self.ws_connection.selected_subprotocol - def get_compression_options(self): + def get_compression_options(self) -> Optional[Dict[str, Any]]: """Override to return compression options for the connection. If this method returns None (the default), compression will @@ -318,7 +399,7 @@ class WebSocketHandler(tornado.web.RequestHandler): # TODO: Add wbits option. return None - def open(self, *args, **kwargs): + def open(self, *args: str, **kwargs: str) -> Optional[Awaitable[None]]: """Invoked when a new WebSocket is opened. The arguments to `open` are extracted from the `tornado.web.URLSpec` @@ -334,7 +415,7 @@ class WebSocketHandler(tornado.web.RequestHandler): """ pass - def on_message(self, message): + def on_message(self, message: Union[str, bytes]) -> Optional[Awaitable[None]]: """Handle incoming messages on the WebSocket This method must be overridden. @@ -345,7 +426,7 @@ class WebSocketHandler(tornado.web.RequestHandler): """ raise NotImplementedError - def ping(self, data=b''): + def ping(self, data: Union[str, bytes] = b"") -> None: """Send ping frame to the remote end. The data argument allows a small amount of data (up to 125 @@ -362,19 +443,19 @@ class WebSocketHandler(tornado.web.RequestHandler): """ data = utf8(data) - if self.ws_connection is None: + if self.ws_connection is None or self.ws_connection.is_closing(): raise WebSocketClosedError() self.ws_connection.write_ping(data) - def on_pong(self, data): + def on_pong(self, data: bytes) -> None: """Invoked when the response to a ping frame is received.""" pass - def on_ping(self, data): + def on_ping(self, data: bytes) -> None: """Invoked when the a ping frame is received.""" pass - def on_close(self): + def on_close(self) -> None: """Invoked when the WebSocket is closed. If the connection was closed cleanly and a status code or reason @@ -387,7 +468,7 @@ class WebSocketHandler(tornado.web.RequestHandler): """ pass - def close(self, code=None, reason=None): + def close(self, code: int = None, reason: str = None) -> None: """Closes this Web Socket. Once the close handshake is successful the socket will be closed. @@ -407,7 +488,7 @@ class WebSocketHandler(tornado.web.RequestHandler): self.ws_connection.close(code, reason) self.ws_connection = None - def check_origin(self, origin): + def check_origin(self, origin: str) -> bool: """Override to enable support for allowing alternate origins. The ``origin`` argument is the value of the ``Origin`` HTTP @@ -417,9 +498,9 @@ class WebSocketHandler(tornado.web.RequestHandler): implement WebSockets support this header, and non-browser clients do not have the same cross-site security concerns). - Should return True to accept the request or False to reject it. - By default, rejects all requests with an origin on a host other - than this one. + Should return ``True`` to accept the request or ``False`` to + reject it. By default, rejects all requests with an origin on + a host other than this one. This is a security protection against cross site scripting attacks on browsers, since WebSockets are allowed to bypass the usual same-origin @@ -439,7 +520,7 @@ class WebSocketHandler(tornado.web.RequestHandler): for more. To accept all cross-origin traffic (which was the default prior to - Tornado 4.0), simply override this method to always return true:: + Tornado 4.0), simply override this method to always return ``True``:: def check_origin(self, origin): return True @@ -463,7 +544,7 @@ class WebSocketHandler(tornado.web.RequestHandler): # Check to see that origin matches host directly, including ports return origin == host - def set_nodelay(self, value): + def set_nodelay(self, value: bool) -> None: """Set the no-delay flag for this stream. By default, small messages may be delayed and/or combined to minimize @@ -477,9 +558,10 @@ class WebSocketHandler(tornado.web.RequestHandler): .. versionadded:: 3.1 """ - self.stream.set_nodelay(value) + assert self.ws_connection is not None + self.ws_connection.set_nodelay(value) - def on_connection_close(self): + def on_connection_close(self) -> None: if self.ws_connection: self.ws_connection.on_connection_close() self.ws_connection = None @@ -488,7 +570,14 @@ class WebSocketHandler(tornado.web.RequestHandler): self.on_close() self._break_cycles() - def _break_cycles(self): + def on_ws_connection_close( + self, close_code: int = None, close_reason: str = None + ) -> None: + self.close_code = close_code + self.close_reason = close_reason + self.on_connection_close() + + def _break_cycles(self) -> None: # WebSocketHandlers call finish() early, but we don't want to # break up reference cycles (which makes it impossible to call # self.render_string) until after we've really closed the @@ -497,7 +586,7 @@ class WebSocketHandler(tornado.web.RequestHandler): if self.get_status() != 101 or self._on_close_called: super(WebSocketHandler, self)._break_cycles() - def send_error(self, *args, **kwargs): + def send_error(self, *args: Any, **kwargs: Any) -> None: if self.stream is None: super(WebSocketHandler, self).send_error(*args, **kwargs) else: @@ -507,36 +596,50 @@ class WebSocketHandler(tornado.web.RequestHandler): # we can close the connection more gracefully. self.stream.close() - def get_websocket_protocol(self): + def get_websocket_protocol(self) -> Optional["WebSocketProtocol"]: websocket_version = self.request.headers.get("Sec-WebSocket-Version") if websocket_version in ("7", "8", "13"): - return WebSocketProtocol13( - self, compression_options=self.get_compression_options()) + params = _WebSocketParams( + ping_interval=self.ping_interval, + ping_timeout=self.ping_timeout, + max_message_size=self.max_message_size, + compression_options=self.get_compression_options(), + ) + return WebSocketProtocol13(self, False, params) + return None - def _attach_stream(self): - self.stream = self.detach() - self.stream.set_close_callback(self.on_connection_close) + def _detach_stream(self) -> IOStream: # disable non-WS methods - for method in ["write", "redirect", "set_header", "set_cookie", - "set_status", "flush", "finish"]: + for method in [ + "write", + "redirect", + "set_header", + "set_cookie", + "set_status", + "flush", + "finish", + ]: setattr(self, method, _raise_not_supported_for_websockets) + return self.detach() -def _raise_not_supported_for_websockets(*args, **kwargs): +def _raise_not_supported_for_websockets(*args: Any, **kwargs: Any) -> None: raise RuntimeError("Method not supported for Web Sockets") -class WebSocketProtocol(object): +class WebSocketProtocol(abc.ABC): """Base class for WebSocket protocol versions. """ - def __init__(self, handler): + + def __init__(self, handler: "_WebSocketDelegate") -> None: self.handler = handler - self.request = handler.request - self.stream = handler.stream + self.stream = None # type: Optional[IOStream] self.client_terminated = False self.server_terminated = False - def _run_callback(self, callback, *args, **kwargs): + def _run_callback( + self, callback: Callable, *args: Any, **kwargs: Any + ) -> "Optional[Future[Any]]": """Runs the given callback with exception handling. If the callback is a coroutine, returns its Future. On error, aborts the @@ -547,80 +650,156 @@ class WebSocketProtocol(object): except Exception: self.handler.log_exception(*sys.exc_info()) self._abort() + return None else: if result is not None: result = gen.convert_yielded(result) + assert self.stream is not None self.stream.io_loop.add_future(result, lambda f: f.result()) return result - def on_connection_close(self): + def on_connection_close(self) -> None: self._abort() - def _abort(self): + def _abort(self) -> None: """Instantly aborts the WebSocket connection by closing the socket""" self.client_terminated = True self.server_terminated = True - self.stream.close() # forcibly tear down the connection + if self.stream is not None: + self.stream.close() # forcibly tear down the connection self.close() # let the subclass cleanup + @abc.abstractmethod + def close(self, code: int = None, reason: str = None) -> None: + raise NotImplementedError() + + @abc.abstractmethod + def is_closing(self) -> bool: + raise NotImplementedError() + + @abc.abstractmethod + async def accept_connection(self, handler: WebSocketHandler) -> None: + raise NotImplementedError() + + @abc.abstractmethod + def write_message( + self, message: Union[str, bytes], binary: bool = False + ) -> "Future[None]": + raise NotImplementedError() + + @property + @abc.abstractmethod + def selected_subprotocol(self) -> Optional[str]: + raise NotImplementedError() + + @abc.abstractmethod + def write_ping(self, data: bytes) -> None: + raise NotImplementedError() + + # The entry points below are used by WebSocketClientConnection, + # which was introduced after we only supported a single version of + # WebSocketProtocol. The WebSocketProtocol/WebSocketProtocol13 + # boundary is currently pretty ad-hoc. + @abc.abstractmethod + def _process_server_headers( + self, key: Union[str, bytes], headers: httputil.HTTPHeaders + ) -> None: + raise NotImplementedError() + + @abc.abstractmethod + def start_pinging(self) -> None: + raise NotImplementedError() + + @abc.abstractmethod + async def _receive_frame_loop(self) -> None: + raise NotImplementedError() + + @abc.abstractmethod + def set_nodelay(self, x: bool) -> None: + raise NotImplementedError() + class _PerMessageDeflateCompressor(object): - def __init__(self, persistent, max_wbits, compression_options=None): + def __init__( + self, + persistent: bool, + max_wbits: Optional[int], + compression_options: Dict[str, Any] = None, + ) -> None: if max_wbits is None: max_wbits = zlib.MAX_WBITS # There is no symbolic constant for the minimum wbits value. if not (8 <= max_wbits <= zlib.MAX_WBITS): - raise ValueError("Invalid max_wbits value %r; allowed range 8-%d", - max_wbits, zlib.MAX_WBITS) + raise ValueError( + "Invalid max_wbits value %r; allowed range 8-%d", + max_wbits, + zlib.MAX_WBITS, + ) self._max_wbits = max_wbits - if compression_options is None or 'compression_level' not in compression_options: + if ( + compression_options is None + or "compression_level" not in compression_options + ): self._compression_level = tornado.web.GZipContentEncoding.GZIP_LEVEL else: - self._compression_level = compression_options['compression_level'] + self._compression_level = compression_options["compression_level"] - if compression_options is None or 'mem_level' not in compression_options: + if compression_options is None or "mem_level" not in compression_options: self._mem_level = 8 else: - self._mem_level = compression_options['mem_level'] + self._mem_level = compression_options["mem_level"] if persistent: - self._compressor = self._create_compressor() + self._compressor = self._create_compressor() # type: Optional[_Compressor] else: self._compressor = None - def _create_compressor(self): - return zlib.compressobj(self._compression_level, - zlib.DEFLATED, -self._max_wbits, self._mem_level) + def _create_compressor(self) -> "_Compressor": + return zlib.compressobj( + self._compression_level, zlib.DEFLATED, -self._max_wbits, self._mem_level + ) - def compress(self, data): + def compress(self, data: bytes) -> bytes: compressor = self._compressor or self._create_compressor() - data = (compressor.compress(data) + - compressor.flush(zlib.Z_SYNC_FLUSH)) - assert data.endswith(b'\x00\x00\xff\xff') + data = compressor.compress(data) + compressor.flush(zlib.Z_SYNC_FLUSH) + assert data.endswith(b"\x00\x00\xff\xff") return data[:-4] class _PerMessageDeflateDecompressor(object): - def __init__(self, persistent, max_wbits, max_message_size, compression_options=None): + def __init__( + self, + persistent: bool, + max_wbits: Optional[int], + max_message_size: int, + compression_options: Dict[str, Any] = None, + ) -> None: self._max_message_size = max_message_size if max_wbits is None: max_wbits = zlib.MAX_WBITS if not (8 <= max_wbits <= zlib.MAX_WBITS): - raise ValueError("Invalid max_wbits value %r; allowed range 8-%d", - max_wbits, zlib.MAX_WBITS) + raise ValueError( + "Invalid max_wbits value %r; allowed range 8-%d", + max_wbits, + zlib.MAX_WBITS, + ) self._max_wbits = max_wbits if persistent: - self._decompressor = self._create_decompressor() + self._decompressor = ( + self._create_decompressor() + ) # type: Optional[_Decompressor] else: self._decompressor = None - def _create_decompressor(self): + def _create_decompressor(self) -> "_Decompressor": return zlib.decompressobj(-self._max_wbits) - def decompress(self, data): + def decompress(self, data: bytes) -> bytes: decompressor = self._decompressor or self._create_decompressor() - result = decompressor.decompress(data + b'\x00\x00\xff\xff', self._max_message_size) + result = decompressor.decompress( + data + b"\x00\x00\xff\xff", self._max_message_size + ) if decompressor.unconsumed_tail: raise _DecompressTooLargeError() return result @@ -632,30 +811,38 @@ class WebSocketProtocol13(WebSocketProtocol): This class supports versions 7 and 8 of the protocol in addition to the final version 13. """ + # Bit masks for the first byte of a frame. FIN = 0x80 RSV1 = 0x40 RSV2 = 0x20 RSV3 = 0x10 RSV_MASK = RSV1 | RSV2 | RSV3 - OPCODE_MASK = 0x0f + OPCODE_MASK = 0x0F - def __init__(self, handler, mask_outgoing=False, - compression_options=None): + stream = None # type: IOStream + + def __init__( + self, + handler: "_WebSocketDelegate", + mask_outgoing: bool, + params: _WebSocketParams, + ) -> None: WebSocketProtocol.__init__(self, handler) self.mask_outgoing = mask_outgoing + self.params = params self._final_frame = False self._frame_opcode = None self._masked_frame = None - self._frame_mask = None + self._frame_mask = None # type: Optional[bytes] self._frame_length = None - self._fragmented_message_buffer = None + self._fragmented_message_buffer = None # type: Optional[bytes] self._fragmented_message_opcode = None - self._waiting = None - self._compression_options = compression_options - self._decompressor = None - self._compressor = None - self._frame_compressed = None + self._waiting = None # type: object + self._compression_options = params.compression_options + self._decompressor = None # type: Optional[_PerMessageDeflateDecompressor] + self._compressor = None # type: Optional[_PerMessageDeflateCompressor] + self._frame_compressed = None # type: Optional[bool] # The total uncompressed size of all messages received or sent. # Unicode messages are encoded to utf8. # Only for testing; subject to change. @@ -665,40 +852,53 @@ class WebSocketProtocol13(WebSocketProtocol): # the effect of compression, frame overhead, and control frames. self._wire_bytes_in = 0 self._wire_bytes_out = 0 - self.ping_callback = None - self.last_ping = 0 - self.last_pong = 0 + self.ping_callback = None # type: Optional[PeriodicCallback] + self.last_ping = 0.0 + self.last_pong = 0.0 + self.close_code = None # type: Optional[int] + self.close_reason = None # type: Optional[str] - def accept_connection(self): + # Use a property for this to satisfy the abc. + @property + def selected_subprotocol(self) -> Optional[str]: + return self._selected_subprotocol + + @selected_subprotocol.setter + def selected_subprotocol(self, value: Optional[str]) -> None: + self._selected_subprotocol = value + + async def accept_connection(self, handler: WebSocketHandler) -> None: try: - self._handle_websocket_headers() + self._handle_websocket_headers(handler) except ValueError: - self.handler.set_status(400) + handler.set_status(400) log_msg = "Missing/Invalid WebSocket headers" - self.handler.finish(log_msg) + handler.finish(log_msg) gen_log.debug(log_msg) return try: - self._accept_connection() + await self._accept_connection(handler) + except asyncio.CancelledError: + self._abort() + return except ValueError: - gen_log.debug("Malformed WebSocket request received", - exc_info=True) + gen_log.debug("Malformed WebSocket request received", exc_info=True) self._abort() return - def _handle_websocket_headers(self): + def _handle_websocket_headers(self, handler: WebSocketHandler) -> None: """Verifies all invariant- and required headers If a header is missing or have an incorrect value ValueError will be raised """ fields = ("Host", "Sec-Websocket-Key", "Sec-Websocket-Version") - if not all(map(lambda f: self.request.headers.get(f), fields)): + if not all(map(lambda f: handler.request.headers.get(f), fields)): raise ValueError("Missing/Invalid WebSocket headers") @staticmethod - def compute_accept_value(key): + def compute_accept_value(key: Union[str, bytes]) -> str: """Computes the value for the Sec-WebSocket-Accept header, given the value for Sec-WebSocket-Key. """ @@ -707,114 +907,143 @@ class WebSocketProtocol13(WebSocketProtocol): sha1.update(b"258EAFA5-E914-47DA-95CA-C5AB0DC85B11") # Magic value return native_str(base64.b64encode(sha1.digest())) - def _challenge_response(self): + def _challenge_response(self, handler: WebSocketHandler) -> str: return WebSocketProtocol13.compute_accept_value( - self.request.headers.get("Sec-Websocket-Key")) + cast(str, handler.request.headers.get("Sec-Websocket-Key")) + ) - @gen.coroutine - def _accept_connection(self): - subprotocol_header = self.request.headers.get("Sec-WebSocket-Protocol") + async def _accept_connection(self, handler: WebSocketHandler) -> None: + subprotocol_header = handler.request.headers.get("Sec-WebSocket-Protocol") if subprotocol_header: - subprotocols = [s.strip() for s in subprotocol_header.split(',')] + subprotocols = [s.strip() for s in subprotocol_header.split(",")] else: subprotocols = [] - self.selected_subprotocol = self.handler.select_subprotocol(subprotocols) + self.selected_subprotocol = handler.select_subprotocol(subprotocols) if self.selected_subprotocol: assert self.selected_subprotocol in subprotocols - self.handler.set_header("Sec-WebSocket-Protocol", self.selected_subprotocol) + handler.set_header("Sec-WebSocket-Protocol", self.selected_subprotocol) - extensions = self._parse_extensions_header(self.request.headers) + extensions = self._parse_extensions_header(handler.request.headers) for ext in extensions: - if (ext[0] == 'permessage-deflate' and - self._compression_options is not None): + if ext[0] == "permessage-deflate" and self._compression_options is not None: # TODO: negotiate parameters if compression_options # specifies limits. - self._create_compressors('server', ext[1], self._compression_options) - if ('client_max_window_bits' in ext[1] and - ext[1]['client_max_window_bits'] is None): + self._create_compressors("server", ext[1], self._compression_options) + if ( + "client_max_window_bits" in ext[1] + and ext[1]["client_max_window_bits"] is None + ): # Don't echo an offered client_max_window_bits # parameter with no value. - del ext[1]['client_max_window_bits'] - self.handler.set_header("Sec-WebSocket-Extensions", - httputil._encode_header( - 'permessage-deflate', ext[1])) + del ext[1]["client_max_window_bits"] + handler.set_header( + "Sec-WebSocket-Extensions", + httputil._encode_header("permessage-deflate", ext[1]), + ) break - self.handler.clear_header("Content-Type") - self.handler.set_status(101) - self.handler.set_header("Upgrade", "websocket") - self.handler.set_header("Connection", "Upgrade") - self.handler.set_header("Sec-WebSocket-Accept", self._challenge_response()) - self.handler.finish() + handler.clear_header("Content-Type") + handler.set_status(101) + handler.set_header("Upgrade", "websocket") + handler.set_header("Connection", "Upgrade") + handler.set_header("Sec-WebSocket-Accept", self._challenge_response(handler)) + handler.finish() - self.handler._attach_stream() - self.stream = self.handler.stream + self.stream = handler._detach_stream() self.start_pinging() - open_result = self._run_callback(self.handler.open, *self.handler.open_args, - **self.handler.open_kwargs) - if open_result is not None: - yield open_result - yield self._receive_frame_loop() + try: + open_result = handler.open(*handler.open_args, **handler.open_kwargs) + if open_result is not None: + await open_result + except Exception: + handler.log_exception(*sys.exc_info()) + self._abort() + return - def _parse_extensions_header(self, headers): - extensions = headers.get("Sec-WebSocket-Extensions", '') + await self._receive_frame_loop() + + def _parse_extensions_header( + self, headers: httputil.HTTPHeaders + ) -> List[Tuple[str, Dict[str, str]]]: + extensions = headers.get("Sec-WebSocket-Extensions", "") if extensions: - return [httputil._parse_header(e.strip()) - for e in extensions.split(',')] + return [httputil._parse_header(e.strip()) for e in extensions.split(",")] return [] - def _process_server_headers(self, key, headers): + def _process_server_headers( + self, key: Union[str, bytes], headers: httputil.HTTPHeaders + ) -> None: """Process the headers sent by the server to this client connection. 'key' is the websocket handshake challenge/response key. """ - assert headers['Upgrade'].lower() == 'websocket' - assert headers['Connection'].lower() == 'upgrade' + assert headers["Upgrade"].lower() == "websocket" + assert headers["Connection"].lower() == "upgrade" accept = self.compute_accept_value(key) - assert headers['Sec-Websocket-Accept'] == accept + assert headers["Sec-Websocket-Accept"] == accept extensions = self._parse_extensions_header(headers) for ext in extensions: - if (ext[0] == 'permessage-deflate' and - self._compression_options is not None): - self._create_compressors('client', ext[1]) + if ext[0] == "permessage-deflate" and self._compression_options is not None: + self._create_compressors("client", ext[1]) else: raise ValueError("unsupported extension %r", ext) - self.selected_subprotocol = headers.get('Sec-WebSocket-Protocol', None) + self.selected_subprotocol = headers.get("Sec-WebSocket-Protocol", None) - def _get_compressor_options(self, side, agreed_parameters, compression_options=None): + def _get_compressor_options( + self, + side: str, + agreed_parameters: Dict[str, Any], + compression_options: Dict[str, Any] = None, + ) -> Dict[str, Any]: """Converts a websocket agreed_parameters set to keyword arguments for our compressor objects. """ options = dict( - persistent=(side + '_no_context_takeover') not in agreed_parameters) - wbits_header = agreed_parameters.get(side + '_max_window_bits', None) + persistent=(side + "_no_context_takeover") not in agreed_parameters + ) # type: Dict[str, Any] + wbits_header = agreed_parameters.get(side + "_max_window_bits", None) if wbits_header is None: - options['max_wbits'] = zlib.MAX_WBITS + options["max_wbits"] = zlib.MAX_WBITS else: - options['max_wbits'] = int(wbits_header) - options['compression_options'] = compression_options + options["max_wbits"] = int(wbits_header) + options["compression_options"] = compression_options return options - def _create_compressors(self, side, agreed_parameters, compression_options=None): + def _create_compressors( + self, + side: str, + agreed_parameters: Dict[str, Any], + compression_options: Dict[str, Any] = None, + ) -> None: # TODO: handle invalid parameters gracefully - allowed_keys = set(['server_no_context_takeover', - 'client_no_context_takeover', - 'server_max_window_bits', - 'client_max_window_bits']) + allowed_keys = set( + [ + "server_no_context_takeover", + "client_no_context_takeover", + "server_max_window_bits", + "client_max_window_bits", + ] + ) for key in agreed_parameters: if key not in allowed_keys: raise ValueError("unsupported compression parameter %r" % key) - other_side = 'client' if (side == 'server') else 'server' + other_side = "client" if (side == "server") else "server" self._compressor = _PerMessageDeflateCompressor( - **self._get_compressor_options(side, agreed_parameters, compression_options)) + **self._get_compressor_options(side, agreed_parameters, compression_options) + ) self._decompressor = _PerMessageDeflateDecompressor( - max_message_size=self.handler.max_message_size, - **self._get_compressor_options(other_side, agreed_parameters, compression_options)) + max_message_size=self.params.max_message_size, + **self._get_compressor_options( + other_side, agreed_parameters, compression_options + ) + ) - def _write_frame(self, fin, opcode, data, flags=0): + def _write_frame( + self, fin: bool, opcode: int, data: bytes, flags: int = 0 + ) -> "Future[None]": data_len = len(data) if opcode & 0x8: # All control frames MUST have a payload length of 125 @@ -845,7 +1074,9 @@ class WebSocketProtocol13(WebSocketProtocol): self._wire_bytes_out += len(frame) return self.stream.write(frame) - def write_message(self, message, binary=False): + def write_message( + self, message: Union[str, bytes], binary: bool = False + ) -> "Future[None]": """Sends the given message to the client of this Web Socket.""" if binary: opcode = 0x2 @@ -868,35 +1099,35 @@ class WebSocketProtocol13(WebSocketProtocol): except StreamClosedError: raise WebSocketClosedError() - @gen.coroutine - def wrapper(): + async def wrapper() -> None: try: - yield fut + await fut except StreamClosedError: raise WebSocketClosedError() - return wrapper() - def write_ping(self, data): + return asyncio.ensure_future(wrapper()) + + def write_ping(self, data: bytes) -> None: """Send ping frame.""" assert isinstance(data, bytes) self._write_frame(True, 0x9, data) - @gen.coroutine - def _receive_frame_loop(self): + async def _receive_frame_loop(self) -> None: try: while not self.client_terminated: - yield self._receive_frame() + await self._receive_frame() except StreamClosedError: self._abort() + self.handler.on_ws_connection_close(self.close_code, self.close_reason) - def _read_bytes(self, n): + async def _read_bytes(self, n: int) -> bytes: + data = await self.stream.read_bytes(n) self._wire_bytes_in += n - return self.stream.read_bytes(n) + return data - @gen.coroutine - def _receive_frame(self): + async def _receive_frame(self) -> None: # Read the frame header. - data = yield self._read_bytes(2) + data = await self._read_bytes(2) header, mask_payloadlen = struct.unpack("BB", data) is_final_frame = header & self.FIN reserved_bits = header & self.RSV_MASK @@ -913,7 +1144,7 @@ class WebSocketProtocol13(WebSocketProtocol): self._abort() return is_masked = bool(mask_payloadlen & 0x80) - payloadlen = mask_payloadlen & 0x7f + payloadlen = mask_payloadlen & 0x7F # Parse and validate the length. if opcode_is_control and payloadlen >= 126: @@ -923,24 +1154,25 @@ class WebSocketProtocol13(WebSocketProtocol): if payloadlen < 126: self._frame_length = payloadlen elif payloadlen == 126: - data = yield self._read_bytes(2) + data = await self._read_bytes(2) payloadlen = struct.unpack("!H", data)[0] elif payloadlen == 127: - data = yield self._read_bytes(8) + data = await self._read_bytes(8) payloadlen = struct.unpack("!Q", data)[0] new_len = payloadlen if self._fragmented_message_buffer is not None: new_len += len(self._fragmented_message_buffer) - if new_len > self.handler.max_message_size: + if new_len > self.params.max_message_size: self.close(1009, "message too big") self._abort() return # Read the payload, unmasking if necessary. if is_masked: - self._frame_mask = yield self._read_bytes(4) - data = yield self._read_bytes(payloadlen) + self._frame_mask = await self._read_bytes(4) + data = await self._read_bytes(payloadlen) if is_masked: + assert self._frame_mask is not None data = _websocket_mask(self._frame_mask, data) # Decide what to do with this frame. @@ -974,20 +1206,21 @@ class WebSocketProtocol13(WebSocketProtocol): if is_final_frame: handled_future = self._handle_message(opcode, data) if handled_future is not None: - yield handled_future + await handled_future - def _handle_message(self, opcode, data): + def _handle_message(self, opcode: int, data: bytes) -> "Optional[Future[None]]": """Execute on_message, returning its Future if it is a coroutine.""" if self.client_terminated: - return + return None if self._frame_compressed: + assert self._decompressor is not None try: data = self._decompressor.decompress(data) except _DecompressTooLargeError: self.close(1009, "message too big after decompression") self._abort() - return + return None if opcode == 0x1: # UTF-8 data @@ -996,7 +1229,7 @@ class WebSocketProtocol13(WebSocketProtocol): decoded = data.decode("utf-8") except UnicodeDecodeError: self._abort() - return + return None return self._run_callback(self.handler.on_message, decoded) elif opcode == 0x2: # Binary data @@ -1006,11 +1239,11 @@ class WebSocketProtocol13(WebSocketProtocol): # Close self.client_terminated = True if len(data) >= 2: - self.handler.close_code = struct.unpack('>H', data[:2])[0] + self.close_code = struct.unpack(">H", data[:2])[0] if len(data) > 2: - self.handler.close_reason = to_unicode(data[2:]) + self.close_reason = to_unicode(data[2:]) # Echo the received close code, if any (RFC 6455 section 5.5.1). - self.close(self.handler.close_code) + self.close(self.close_code) elif opcode == 0x9: # Ping try: @@ -1024,17 +1257,18 @@ class WebSocketProtocol13(WebSocketProtocol): return self._run_callback(self.handler.on_pong, data) else: self._abort() + return None - def close(self, code=None, reason=None): + def close(self, code: int = None, reason: str = None) -> None: """Closes the WebSocket connection.""" if not self.server_terminated: if not self.stream.closed(): if code is None and reason is not None: code = 1000 # "normal closure" status code if code is None: - close_data = b'' + close_data = b"" else: - close_data = struct.pack('>H', code) + close_data = struct.pack(">H", code) if reason is not None: close_data += utf8(reason) try: @@ -1051,36 +1285,49 @@ class WebSocketProtocol13(WebSocketProtocol): # Give the client a few seconds to complete a clean shutdown, # otherwise just close the connection. self._waiting = self.stream.io_loop.add_timeout( - self.stream.io_loop.time() + 5, self._abort) + self.stream.io_loop.time() + 5, self._abort + ) + + def is_closing(self) -> bool: + """Return ``True`` if this connection is closing. + + The connection is considered closing if either side has + initiated its closing handshake or if the stream has been + shut down uncleanly. + """ + return self.stream.closed() or self.client_terminated or self.server_terminated @property - def ping_interval(self): - interval = self.handler.ping_interval + def ping_interval(self) -> Optional[float]: + interval = self.params.ping_interval if interval is not None: return interval return 0 @property - def ping_timeout(self): - timeout = self.handler.ping_timeout + def ping_timeout(self) -> Optional[float]: + timeout = self.params.ping_timeout if timeout is not None: return timeout + assert self.ping_interval is not None return max(3 * self.ping_interval, 30) - def start_pinging(self): + def start_pinging(self) -> None: """Start sending periodic pings to keep the connection alive""" + assert self.ping_interval is not None if self.ping_interval > 0: self.last_ping = self.last_pong = IOLoop.current().time() self.ping_callback = PeriodicCallback( - self.periodic_ping, self.ping_interval * 1000) + self.periodic_ping, self.ping_interval * 1000 + ) self.ping_callback.start() - def periodic_ping(self): + def periodic_ping(self) -> None: """Send a ping to keep the websocket alive Called periodically if the websocket_ping_interval is set and non-zero. """ - if self.stream.closed() and self.ping_callback is not None: + if self.is_closing() and self.ping_callback is not None: self.ping_callback.stop() return @@ -1090,14 +1337,21 @@ class WebSocketProtocol13(WebSocketProtocol): now = IOLoop.current().time() since_last_pong = now - self.last_pong since_last_ping = now - self.last_ping - if (since_last_ping < 2 * self.ping_interval and - since_last_pong > self.ping_timeout): + assert self.ping_interval is not None + assert self.ping_timeout is not None + if ( + since_last_ping < 2 * self.ping_interval + and since_last_pong > self.ping_timeout + ): self.close() return - self.write_ping(b'') + self.write_ping(b"") self.last_ping = now + def set_nodelay(self, x: bool) -> None: + self.stream.set_nodelay(x) + class WebSocketClientConnection(simple_httpclient._HTTPConnection): """WebSocket client connection. @@ -1105,46 +1359,68 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection): This class should not be instantiated directly; use the `websocket_connect` function instead. """ - def __init__(self, request, on_message_callback=None, - compression_options=None, ping_interval=None, ping_timeout=None, - max_message_size=None, subprotocols=[]): - self.compression_options = compression_options - self.connect_future = Future() - self.protocol = None - self.read_queue = Queue(1) + + protocol = None # type: WebSocketProtocol + + def __init__( + self, + request: httpclient.HTTPRequest, + on_message_callback: Callable[[Union[None, str, bytes]], None] = None, + compression_options: Dict[str, Any] = None, + ping_interval: float = None, + ping_timeout: float = None, + max_message_size: int = _default_max_message_size, + subprotocols: Optional[List[str]] = [], + ) -> None: + self.connect_future = Future() # type: Future[WebSocketClientConnection] + self.read_queue = Queue(1) # type: Queue[Union[None, str, bytes]] self.key = base64.b64encode(os.urandom(16)) self._on_message_callback = on_message_callback - self.close_code = self.close_reason = None - self.ping_interval = ping_interval - self.ping_timeout = ping_timeout - self.max_message_size = max_message_size + self.close_code = None # type: Optional[int] + self.close_reason = None # type: Optional[str] + self.params = _WebSocketParams( + ping_interval=ping_interval, + ping_timeout=ping_timeout, + max_message_size=max_message_size, + compression_options=compression_options, + ) - scheme, sep, rest = request.url.partition(':') - scheme = {'ws': 'http', 'wss': 'https'}[scheme] + scheme, sep, rest = request.url.partition(":") + scheme = {"ws": "http", "wss": "https"}[scheme] request.url = scheme + sep + rest - request.headers.update({ - 'Upgrade': 'websocket', - 'Connection': 'Upgrade', - 'Sec-WebSocket-Key': self.key, - 'Sec-WebSocket-Version': '13', - }) + request.headers.update( + { + "Upgrade": "websocket", + "Connection": "Upgrade", + "Sec-WebSocket-Key": self.key, + "Sec-WebSocket-Version": "13", + } + ) if subprotocols is not None: - request.headers['Sec-WebSocket-Protocol'] = ','.join(subprotocols) - if self.compression_options is not None: + request.headers["Sec-WebSocket-Protocol"] = ",".join(subprotocols) + if compression_options is not None: # Always offer to let the server set our max_wbits (and even though # we don't offer it, we will accept a client_no_context_takeover # from the server). # TODO: set server parameters for deflate extension # if requested in self.compression_options. - request.headers['Sec-WebSocket-Extensions'] = ( - 'permessage-deflate; client_max_window_bits') + request.headers[ + "Sec-WebSocket-Extensions" + ] = "permessage-deflate; client_max_window_bits" self.tcp_client = TCPClient() super(WebSocketClientConnection, self).__init__( - None, request, lambda: None, self._on_http_response, - 104857600, self.tcp_client, 65536, 104857600) + None, + request, + lambda: None, + self._on_http_response, + 104857600, + self.tcp_client, + 65536, + 104857600, + ) - def close(self, code=None, reason=None): + def close(self, code: int = None, reason: str = None) -> None: """Closes the websocket connection. ``code`` and ``reason`` are documented under @@ -1158,49 +1434,66 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection): """ if self.protocol is not None: self.protocol.close(code, reason) - self.protocol = None + self.protocol = None # type: ignore - def on_connection_close(self): + def on_connection_close(self) -> None: if not self.connect_future.done(): self.connect_future.set_exception(StreamClosedError()) - self.on_message(None) + self._on_message(None) self.tcp_client.close() super(WebSocketClientConnection, self).on_connection_close() - def _on_http_response(self, response): + def on_ws_connection_close( + self, close_code: int = None, close_reason: str = None + ) -> None: + self.close_code = close_code + self.close_reason = close_reason + self.on_connection_close() + + def _on_http_response(self, response: httpclient.HTTPResponse) -> None: if not self.connect_future.done(): if response.error: self.connect_future.set_exception(response.error) else: - self.connect_future.set_exception(WebSocketError( - "Non-websocket response")) + self.connect_future.set_exception( + WebSocketError("Non-websocket response") + ) - def headers_received(self, start_line, headers): + async def headers_received( + self, + start_line: Union[httputil.RequestStartLine, httputil.ResponseStartLine], + headers: httputil.HTTPHeaders, + ) -> None: + assert isinstance(start_line, httputil.ResponseStartLine) if start_line.code != 101: - return super(WebSocketClientConnection, self).headers_received( - start_line, headers) - - self.headers = headers - self.protocol = self.get_websocket_protocol() - self.protocol._process_server_headers(self.key, self.headers) - self.protocol.start_pinging() - IOLoop.current().add_callback(self.protocol._receive_frame_loop) + await super(WebSocketClientConnection, self).headers_received( + start_line, headers + ) + return if self._timeout is not None: self.io_loop.remove_timeout(self._timeout) self._timeout = None - self.stream = self.connection.detach() - self.stream.set_close_callback(self.on_connection_close) + self.headers = headers + self.protocol = self.get_websocket_protocol() + self.protocol._process_server_headers(self.key, self.headers) + self.protocol.stream = self.connection.detach() + + IOLoop.current().add_callback(self.protocol._receive_frame_loop) + self.protocol.start_pinging() + # Once we've taken over the connection, clear the final callback # we set on the http request. This deactivates the error handling # in simple_httpclient that would otherwise interfere with our # ability to see exceptions. - self.final_callback = None + self.final_callback = None # type: ignore future_set_result_unless_cancelled(self.connect_future, self) - def write_message(self, message, binary=False): + def write_message( + self, message: Union[str, bytes], binary: bool = False + ) -> "Future[None]": """Sends a message to the WebSocket server. If the stream is closed, raises `WebSocketClosedError`. @@ -1212,7 +1505,9 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection): """ return self.protocol.write_message(message, binary=binary) - def read_message(self, callback=None): + def read_message( + self, callback: Callable[["Future[Union[None, str, bytes]]"], None] = None + ) -> Awaitable[Union[None, str, bytes]]: """Reads a message from the WebSocket server. If on_message_callback was specified at WebSocket @@ -1224,18 +1519,24 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection): ready. """ - future = self.read_queue.get() + awaitable = self.read_queue.get() if callback is not None: - self.io_loop.add_future(future, callback) - return future + self.io_loop.add_future(asyncio.ensure_future(awaitable), callback) + return awaitable - def on_message(self, message): + def on_message(self, message: Union[str, bytes]) -> Optional[Awaitable[None]]: + return self._on_message(message) + + def _on_message( + self, message: Union[None, str, bytes] + ) -> Optional[Awaitable[None]]: if self._on_message_callback: self._on_message_callback(message) + return None else: return self.read_queue.put(message) - def ping(self, data=b''): + def ping(self, data: bytes = b"") -> None: """Send ping frame to the remote end. The data argument allows a small amount of data (up to 125 @@ -1254,29 +1555,45 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection): raise WebSocketClosedError() self.protocol.write_ping(data) - def on_pong(self, data): + def on_pong(self, data: bytes) -> None: pass - def on_ping(self, data): + def on_ping(self, data: bytes) -> None: pass - def get_websocket_protocol(self): - return WebSocketProtocol13(self, mask_outgoing=True, - compression_options=self.compression_options) + def get_websocket_protocol(self) -> WebSocketProtocol: + return WebSocketProtocol13(self, mask_outgoing=True, params=self.params) @property - def selected_subprotocol(self): + def selected_subprotocol(self) -> Optional[str]: """The subprotocol selected by the server. .. versionadded:: 5.1 """ return self.protocol.selected_subprotocol + def log_exception( + self, + typ: "Optional[Type[BaseException]]", + value: Optional[BaseException], + tb: Optional[TracebackType], + ) -> None: + assert typ is not None + assert value is not None + app_log.error("Uncaught exception %s", value, exc_info=(typ, value, tb)) -def websocket_connect(url, callback=None, connect_timeout=None, - on_message_callback=None, compression_options=None, - ping_interval=None, ping_timeout=None, - max_message_size=_default_max_message_size, subprotocols=None): + +def websocket_connect( + url: Union[str, httpclient.HTTPRequest], + callback: Callable[["Future[WebSocketClientConnection]"], None] = None, + connect_timeout: float = None, + on_message_callback: Callable[[Union[None, str, bytes]], None] = None, + compression_options: Dict[str, Any] = None, + ping_interval: float = None, + ping_timeout: float = None, + max_message_size: int = _default_max_message_size, + subprotocols: List[str] = None, +) -> "Awaitable[WebSocketClientConnection]": """Client-side websocket support. Takes a url and returns a Future whose result is a @@ -1328,15 +1645,19 @@ def websocket_connect(url, callback=None, connect_timeout=None, request.headers = httputil.HTTPHeaders(request.headers) else: request = httpclient.HTTPRequest(url, connect_timeout=connect_timeout) - request = httpclient._RequestProxy( - request, httpclient.HTTPRequest._DEFAULTS) - conn = WebSocketClientConnection(request, - on_message_callback=on_message_callback, - compression_options=compression_options, - ping_interval=ping_interval, - ping_timeout=ping_timeout, - max_message_size=max_message_size, - subprotocols=subprotocols) + request = cast( + httpclient.HTTPRequest, + httpclient._RequestProxy(request, httpclient.HTTPRequest._DEFAULTS), + ) + conn = WebSocketClientConnection( + request, + on_message_callback=on_message_callback, + compression_options=compression_options, + ping_interval=ping_interval, + ping_timeout=ping_timeout, + max_message_size=max_message_size, + subprotocols=subprotocols, + ) if callback is not None: IOLoop.current().add_future(conn.connect_future, callback) return conn.connect_future diff --git a/server/www/packages/packages-linux/x64/tornado/wsgi.py b/server/www/packages/packages-linux/x64/tornado/wsgi.py index e1230da..77124aa 100644 --- a/server/www/packages/packages-linux/x64/tornado/wsgi.py +++ b/server/www/packages/packages-linux/x64/tornado/wsgi.py @@ -16,235 +16,41 @@ """WSGI support for the Tornado web framework. WSGI is the Python standard for web servers, and allows for interoperability -between Tornado and other Python web frameworks and servers. This module -provides WSGI support in two ways: +between Tornado and other Python web frameworks and servers. + +This module provides WSGI support via the `WSGIContainer` class, which +makes it possible to run applications using other WSGI frameworks on +the Tornado HTTP server. The reverse is not supported; the Tornado +`.Application` and `.RequestHandler` classes are designed for use with +the Tornado `.HTTPServer` and cannot be used in a generic WSGI +container. -* `WSGIAdapter` converts a `tornado.web.Application` to the WSGI application - interface. This is useful for running a Tornado app on another - HTTP server, such as Google App Engine. See the `WSGIAdapter` class - documentation for limitations that apply. -* `WSGIContainer` lets you run other WSGI applications and frameworks on the - Tornado HTTP server. For example, with this class you can mix Django - and Tornado handlers in a single server. """ -from __future__ import absolute_import, division, print_function - import sys from io import BytesIO import tornado -import warnings -from tornado.concurrent import Future from tornado import escape from tornado import httputil from tornado.log import access_log -from tornado import web -from tornado.escape import native_str -from tornado.util import unicode_type, PY3 +from typing import List, Tuple, Optional, Callable, Any, Dict, Text +from types import TracebackType +import typing + +if typing.TYPE_CHECKING: + from typing import Type # noqa: F401 + from wsgiref.types import WSGIApplication as WSGIAppType # noqa: F401 -if PY3: - import urllib.parse as urllib_parse # py3 -else: - import urllib as urllib_parse # PEP 3333 specifies that WSGI on python 3 generally deals with byte strings # that are smuggled inside objects of type unicode (via the latin1 encoding). -# These functions are like those in the tornado.escape module, but defined -# here to minimize the temptation to use them in non-wsgi contexts. -if str is unicode_type: - def to_wsgi_str(s): - assert isinstance(s, bytes) - return s.decode('latin1') - - def from_wsgi_str(s): - assert isinstance(s, str) - return s.encode('latin1') -else: - def to_wsgi_str(s): - assert isinstance(s, bytes) - return s - - def from_wsgi_str(s): - assert isinstance(s, str) - return s - - -class WSGIApplication(web.Application): - """A WSGI equivalent of `tornado.web.Application`. - - .. deprecated:: 4.0 - - Use a regular `.Application` and wrap it in `WSGIAdapter` instead. - This class will be removed in Tornado 6.0. - """ - def __call__(self, environ, start_response): - return WSGIAdapter(self)(environ, start_response) - - -# WSGI has no facilities for flow control, so just return an already-done -# Future when the interface requires it. -def _dummy_future(): - f = Future() - f.set_result(None) - return f - - -class _WSGIConnection(httputil.HTTPConnection): - def __init__(self, method, start_response, context): - self.method = method - self.start_response = start_response - self.context = context - self._write_buffer = [] - self._finished = False - self._expected_content_remaining = None - self._error = None - - def set_close_callback(self, callback): - # WSGI has no facility for detecting a closed connection mid-request, - # so we can simply ignore the callback. - pass - - def write_headers(self, start_line, headers, chunk=None, callback=None): - if self.method == 'HEAD': - self._expected_content_remaining = 0 - elif 'Content-Length' in headers: - self._expected_content_remaining = int(headers['Content-Length']) - else: - self._expected_content_remaining = None - self.start_response( - '%s %s' % (start_line.code, start_line.reason), - [(native_str(k), native_str(v)) for (k, v) in headers.get_all()]) - if chunk is not None: - self.write(chunk, callback) - elif callback is not None: - callback() - return _dummy_future() - - def write(self, chunk, callback=None): - if self._expected_content_remaining is not None: - self._expected_content_remaining -= len(chunk) - if self._expected_content_remaining < 0: - self._error = httputil.HTTPOutputError( - "Tried to write more data than Content-Length") - raise self._error - self._write_buffer.append(chunk) - if callback is not None: - callback() - return _dummy_future() - - def finish(self): - if (self._expected_content_remaining is not None and - self._expected_content_remaining != 0): - self._error = httputil.HTTPOutputError( - "Tried to write %d bytes less than Content-Length" % - self._expected_content_remaining) - raise self._error - self._finished = True - - -class _WSGIRequestContext(object): - def __init__(self, remote_ip, protocol): - self.remote_ip = remote_ip - self.protocol = protocol - - def __str__(self): - return self.remote_ip - - -class WSGIAdapter(object): - """Converts a `tornado.web.Application` instance into a WSGI application. - - Example usage:: - - import tornado.web - import tornado.wsgi - import wsgiref.simple_server - - class MainHandler(tornado.web.RequestHandler): - def get(self): - self.write("Hello, world") - - if __name__ == "__main__": - application = tornado.web.Application([ - (r"/", MainHandler), - ]) - wsgi_app = tornado.wsgi.WSGIAdapter(application) - server = wsgiref.simple_server.make_server('', 8888, wsgi_app) - server.serve_forever() - - See the `appengine demo - `_ - for an example of using this module to run a Tornado app on Google - App Engine. - - In WSGI mode asynchronous methods are not supported. This means - that it is not possible to use `.AsyncHTTPClient`, or the - `tornado.auth` or `tornado.websocket` modules. - - In multithreaded WSGI servers on Python 3, it may be necessary to - permit `asyncio` to create event loops on any thread. Run the - following at startup (typically import time for WSGI - applications):: - - import asyncio - from tornado.platform.asyncio import AnyThreadEventLoopPolicy - asyncio.set_event_loop_policy(AnyThreadEventLoopPolicy()) - - .. versionadded:: 4.0 - - .. deprecated:: 5.1 - - This class is deprecated and will be removed in Tornado 6.0. - Use Tornado's `.HTTPServer` instead of a WSGI container. - """ - def __init__(self, application): - warnings.warn("WSGIAdapter is deprecated, use Tornado's HTTPServer instead", - DeprecationWarning) - if isinstance(application, WSGIApplication): - self.application = lambda request: web.Application.__call__( - application, request) - else: - self.application = application - - def __call__(self, environ, start_response): - method = environ["REQUEST_METHOD"] - uri = urllib_parse.quote(from_wsgi_str(environ.get("SCRIPT_NAME", ""))) - uri += urllib_parse.quote(from_wsgi_str(environ.get("PATH_INFO", ""))) - if environ.get("QUERY_STRING"): - uri += "?" + environ["QUERY_STRING"] - headers = httputil.HTTPHeaders() - if environ.get("CONTENT_TYPE"): - headers["Content-Type"] = environ["CONTENT_TYPE"] - if environ.get("CONTENT_LENGTH"): - headers["Content-Length"] = environ["CONTENT_LENGTH"] - for key in environ: - if key.startswith("HTTP_"): - headers[key[5:].replace("_", "-")] = environ[key] - if headers.get("Content-Length"): - body = environ["wsgi.input"].read( - int(headers["Content-Length"])) - else: - body = b"" - protocol = environ["wsgi.url_scheme"] - remote_ip = environ.get("REMOTE_ADDR", "") - if environ.get("HTTP_HOST"): - host = environ["HTTP_HOST"] - else: - host = environ["SERVER_NAME"] - connection = _WSGIConnection(method, start_response, - _WSGIRequestContext(remote_ip, protocol)) - request = httputil.HTTPServerRequest( - method, uri, "HTTP/1.1", headers=headers, body=body, - host=host, connection=connection) - request._parse_body() - self.application(request) - if connection._error: - raise connection._error - if not connection._finished: - raise Exception("request did not finish synchronously") - return connection._write_buffer +# This function is like those in the tornado.escape module, but defined +# here to minimize the temptation to use it in non-wsgi contexts. +def to_wsgi_str(s: bytes) -> str: + assert isinstance(s, bytes) + return s.decode("latin1") class WSGIContainer(object): @@ -281,31 +87,44 @@ class WSGIContainer(object): Tornado and WSGI apps in the same server. See https://github.com/bdarnell/django-tornado-demo for a complete example. """ - def __init__(self, wsgi_application): + + def __init__(self, wsgi_application: "WSGIAppType") -> None: self.wsgi_application = wsgi_application - def __call__(self, request): - data = {} - response = [] + def __call__(self, request: httputil.HTTPServerRequest) -> None: + data = {} # type: Dict[str, Any] + response = [] # type: List[bytes] - def start_response(status, response_headers, exc_info=None): + def start_response( + status: str, + headers: List[Tuple[str, str]], + exc_info: Optional[ + Tuple[ + "Optional[Type[BaseException]]", + Optional[BaseException], + Optional[TracebackType], + ] + ] = None, + ) -> Callable[[bytes], Any]: data["status"] = status - data["headers"] = response_headers + data["headers"] = headers return response.append + app_response = self.wsgi_application( - WSGIContainer.environ(request), start_response) + WSGIContainer.environ(request), start_response + ) try: response.extend(app_response) body = b"".join(response) finally: if hasattr(app_response, "close"): - app_response.close() + app_response.close() # type: ignore if not data: raise Exception("WSGI app did not call start_response") - status_code, reason = data["status"].split(' ', 1) - status_code = int(status_code) - headers = data["headers"] + status_code_str, reason = data["status"].split(" ", 1) + status_code = int(status_code_str) + headers = data["headers"] # type: List[Tuple[str, str]] header_set = set(k.lower() for (k, v) in headers) body = escape.utf8(body) if status_code != 304: @@ -320,12 +139,13 @@ class WSGIContainer(object): header_obj = httputil.HTTPHeaders() for key, value in headers: header_obj.add(key, value) + assert request.connection is not None request.connection.write_headers(start_line, header_obj, chunk=body) request.connection.finish() self._log(status_code, request) @staticmethod - def environ(request): + def environ(request: httputil.HTTPServerRequest) -> Dict[Text, Any]: """Converts a `tornado.httputil.HTTPServerRequest` to a WSGI environment. """ hostport = request.host.split(":") @@ -338,8 +158,9 @@ class WSGIContainer(object): environ = { "REQUEST_METHOD": request.method, "SCRIPT_NAME": "", - "PATH_INFO": to_wsgi_str(escape.url_unescape( - request.path, encoding=None, plus=False)), + "PATH_INFO": to_wsgi_str( + escape.url_unescape(request.path, encoding=None, plus=False) + ), "QUERY_STRING": request.query, "REMOTE_ADDR": request.remote_ip, "SERVER_NAME": host, @@ -361,7 +182,7 @@ class WSGIContainer(object): environ["HTTP_" + key.replace("-", "_").upper()] = value return environ - def _log(self, status_code, request): + def _log(self, status_code: int, request: httputil.HTTPServerRequest) -> None: if status_code < 400: log_method = access_log.info elif status_code < 500: @@ -369,8 +190,9 @@ class WSGIContainer(object): else: log_method = access_log.error request_time = 1000.0 * request.request_time() - summary = request.method + " " + request.uri + " (" + \ - request.remote_ip + ")" + assert request.method is not None + assert request.uri is not None + summary = request.method + " " + request.uri + " (" + request.remote_ip + ")" log_method("%d %s %.2fms", status_code, summary, request_time)