upgrade to python3.7.5

build/build-py-static.sh

@@ -3,10 +3,10 @@
 ################################################################
 # Basic settings.
 ################################################################
-VER_PYTHON="3.7.4"
+VER_PYTHON="3.7.5"
 VER_PYTHON_SHORT="3.7"
-VER_OPENSSL="1.0.2s"
+VER_OPENSSL="1.1.1d"
-VER_SQLITE="3290000"
+VER_SQLITE="3300100"
 VER_ZLIB="1.2.11"
 VER_PYTHON_LIB="${VER_PYTHON_SHORT}m"
 
@@ -38,15 +38,6 @@ function on_error()
 	exit 1
 }
 
-function setp_build_git()
-{
-	# su -s
-	# yum install zlib-devel expat-devel libcurl-devel
-	# make prefix=/usr/local
-	# make prefix=/usr/local install
-	echo 'skip build git now.'
-}
-
 function dlfile()
 {
 	echo -n "Downloading $1 ..."

build/build.py

@@ -64,6 +64,9 @@ def main():
         elif x == 'a':
             clean_everything()
             continue
+        elif x == 'e':
+            clean_external()
+            continue
 
         try:
             x = int(x)
@@ -117,6 +120,27 @@ def clean_everything():
     utils.remove(os.path.join(env.root_path, 'external', 'linux', 'release', 'lib', 'libuv.a'))
 
 
+def clean_external():
+    #utils.remove(os.path.join(env.root_path, 'out'))
+    utils.remove(os.path.join(env.root_path, 'external', 'jsoncpp'))
+    utils.remove(os.path.join(env.root_path, 'external', 'libuv'))
+    utils.remove(os.path.join(env.root_path, 'external', 'mbedtls'))
+    utils.remove(os.path.join(env.root_path, 'external', 'mongoose'))
+    #utils.remove(os.path.join(env.root_path, 'external', 'openssl'))
+    #utils.remove(os.path.join(env.root_path, 'external', 'python'))
+    #utils.remove(os.path.join(env.root_path, 'external', 'libssh-win-static', 'lib'))
+    #utils.remove(os.path.join(env.root_path, 'external', 'libssh-win-static', 'src'))
+    #utils.remove(os.path.join(env.root_path, 'external', 'linux', 'tmp'))
+    utils.remove(os.path.join(env.root_path, 'external', 'linux', 'release', 'lib', 'libmbedcrypto.a'))
+    utils.remove(os.path.join(env.root_path, 'external', 'linux', 'release', 'lib', 'libmbedtls.a'))
+    utils.remove(os.path.join(env.root_path, 'external', 'linux', 'release', 'lib', 'libmbedx509.a'))
+    utils.remove(os.path.join(env.root_path, 'external', 'linux', 'release', 'lib', 'libsqlite3.a'))
+    utils.remove(os.path.join(env.root_path, 'external', 'linux', 'release', 'lib', 'libssh.a'))
+    utils.remove(os.path.join(env.root_path, 'external', 'linux', 'release', 'lib', 'libssh_threads.a'))
+
+    utils.remove(os.path.join(env.root_path, 'external', 'linux', 'release', 'lib', 'libuv.a'))
+
+
 def do_opt(opt):
     arg = ''
 
@@ -249,6 +273,7 @@ def show_menu():
         cc.o((cc.CR_NORMAL, '  ['), (cc.CR_INFO, '%2d' % options[o]['id']), (cc.CR_NORMAL, '] ', options[o]['disp']))
 
     cc.v('  -------------------------------------------------------')
+    cc.o((cc.CR_NORMAL, '  ['), (cc.CR_INFO, ' E'), (cc.CR_NORMAL, '] clean external temp. files.'))
     cc.o((cc.CR_NORMAL, '  ['), (cc.CR_INFO, ' C'), (cc.CR_NORMAL, '] clean build and dist.'))
     cc.o((cc.CR_NORMAL, '  ['), (cc.CR_INFO, ' A'), (cc.CR_NORMAL, '] clean everything.'))
 

build/builder/build-external.py

@@ -27,8 +27,6 @@ class BuilderBase:
 
     def build_jsoncpp(self):
         file_name = 'jsoncpp-{}.zip'.format(env.ver_jsoncpp)
-        # if not utils.download_file('jsoncpp source tarball', 'https://github.com/open-source-parsers/jsoncpp/archive/{}.zip'.format(env.ver_jsoncpp), PATH_DOWNLOAD, file_name):
-        #     return
         self._build_jsoncpp(file_name)
 
     def _download_jsoncpp(self, file_name):
@@ -39,8 +37,6 @@ class BuilderBase:
 
     def build_mongoose(self):
         file_name = 'mongoose-{}.zip'.format(env.ver_mongoose)
-        # if not utils.download_file('mongoose source tarball', 'https://github.com/cesanta/mongoose/archive/{}.zip'.format(env.ver_mongoose), PATH_DOWNLOAD, file_name):
-        #     return
         self._build_mongoose(file_name)
 
     def _download_mongoose(self, file_name):
@@ -68,8 +64,6 @@ class BuilderBase:
 
     def build_libuv(self):
         file_name = 'libuv-{}.zip'.format(env.ver_libuv)
-        # if not utils.download_file('libuv source tarball', 'https://github.com/libuv/libuv/archive/v{}.zip'.format(env.ver_libuv), PATH_DOWNLOAD, file_name):
-        #     return
         self._build_libuv(file_name)
 
     def _download_libuv(self, file_name):
@@ -80,8 +74,6 @@ class BuilderBase:
 
     def build_mbedtls(self):
         file_name = 'mbedtls-mbedtls-{}.zip'.format(env.ver_mbedtls)
-        # if not utils.download_file('mbedtls source tarball', 'https://github.com/ARMmbed/mbedtls/archive/mbedtls-{}.zip'.format(env.ver_mbedtls), PATH_DOWNLOAD, file_name):
-        #     return
         self._build_mbedtls(file_name)
 
     def _download_mbedtls(self, file_name):
@@ -92,8 +84,6 @@ class BuilderBase:
 
     def build_zlib(self):
         file_name = 'zlilb{}.zip'.format(env.ver_zlib_number)
-        # if not utils.download_file('mbedtls source tarball', 'https://www.zlib.net/zlib{}.zip'.format(env.ver_zlib_number), PATH_DOWNLOAD, file_name):
-        #     return
         self._build_zlib(file_name)
 
     def _download_zlib(self, file_name):
@@ -104,8 +94,6 @@ class BuilderBase:
 
     def build_libssh(self):
         file_name = 'libssh-{}.zip'.format(env.ver_libssh)
-        # if not utils.download_file('libssh source tarball', 'https://git.libssh.org/projects/libssh.git/snapshot/libssh-{}.zip'.format(env.ver_libssh), PATH_DOWNLOAD, file_name):
-        #     return
         self._build_libssh(file_name)
 
     def _download_libssh(self, file_name):
@@ -405,7 +393,7 @@ class BuilderLinux(BuilderBase):
         cc.n('prepare python header and lib files ...')
 
         if os.path.exists(os.path.join(self.PATH_RELEASE, 'include', 'python', 'Python.h')):
-            cc.w(' - header file already exists, skip.')
+            cc.w('python header file already exists, skip.')
         else:
             utils.ensure_file_exists(os.path.join(self.PATH_RELEASE, 'include', 'python{}m'.format(ctx.py_dot_ver), 'Python.h'))
             utils.sys_exec('ln -s "{}" "{}"'.format(
@@ -417,6 +405,8 @@ class BuilderLinux(BuilderBase):
         utils.ensure_file_exists(os.path.join(self.PATH_RELEASE, 'lib', lib_file))
 
     def _build_jsoncpp(self, file_name):
+        if not self._download_jsoncpp(file_name):
+            return
         cc.n('prepare jsoncpp source code...', end='')
         if not os.path.exists(self.JSONCPP_PATH_SRC):
             cc.v('')
@@ -426,6 +416,8 @@ class BuilderLinux(BuilderBase):
             cc.w('already exists, skip.')
 
     def _build_mongoose(self, file_name):
+        if not self._download_mongoose(file_name):
+            return
         cc.n('prepare mongoose source code...', end='')
         if not os.path.exists(self.MONGOOSE_PATH_SRC):
             cc.v('')
@@ -436,9 +428,11 @@ class BuilderLinux(BuilderBase):
 
     def _build_openssl(self, file_name):
         # we do not need build openssl anymore, because first time run build.sh we built Python with openssl included.
-        pass
+        cc.w('skip build openssl again.')
 
     def _build_libuv(self, file_name):
+        if not self._download_libuv(file_name):
+            return
         if not os.path.exists(self.LIBUV_PATH_SRC):
             os.system('unzip "{}/{}" -d "{}"'.format(PATH_DOWNLOAD, file_name, self.PATH_TMP))
 
@@ -468,6 +462,8 @@ class BuilderLinux(BuilderBase):
         utils.ensure_file_exists(os.path.join(self.PATH_RELEASE, 'lib', 'libuv.a'))
 
     def _build_mbedtls(self, file_name):
+        if not self._download_mbedtls(file_name):
+            return
         if not os.path.exists(self.MBEDTLS_PATH_SRC):
             os.system('unzip "{}/{}" -d "{}"'.format(PATH_DOWNLOAD, file_name, self.PATH_TMP))
 
@@ -510,11 +506,14 @@ class BuilderLinux(BuilderBase):
         os.chdir(old_p)
 
     def _build_libssh(self, file_name):
+        if not self._download_libssh(file_name):
+            return
         if not os.path.exists(self.LIBSSH_PATH_SRC):
             os.system('unzip "{}/{}" -d "{}"'.format(PATH_DOWNLOAD, file_name, self.PATH_TMP))
 
         cc.n('build libssh...', end='')
-        if os.path.exists(os.path.join(self.PATH_RELEASE, 'lib', 'libssh.a')):
+        out_file = os.path.join(self.PATH_RELEASE, 'lib64', 'libssh.a')
+        if os.path.exists(out_file):
             cc.w('already exists, skip.')
             return
         cc.v('')
@@ -535,16 +534,19 @@ class BuilderLinux(BuilderBase):
                        ' -DOPENSSL_LIBRARIES={path_release}/lib' \
                        ' -DWITH_SFTP=ON' \
                        ' -DWITH_SERVER=ON' \
-                       ' -DWITH_STATIC_LIB=ON' \
                        ' -DWITH_GSSAPI=OFF' \
-                       ' -DWITH_ZLIB=OFF' \
+                       ' -DWITH_ZLIB=ON' \
                        ' -DWITH_PCAP=OFF' \
+                       ' -DBUILD_SHARED_LIBS=OFF' \
                        ' -DUNIT_TESTING=OFF' \
                        ' -DWITH_EXAMPLES=OFF' \
                        ' -DWITH_BENCHMARKS=OFF' \
                        ' -DWITH_NACL=OFF' \
                        ' ..'.format(path_release=self.PATH_RELEASE)
 
+        # ' -DWITH_STATIC_LIB=ON'
+
+
         old_p = os.getcwd()
         try:
             utils.cmake(build_path, 'Release', False, cmake_define=cmake_define, cmake_pre_define='CFLAGS="-fPIC"')
@@ -554,12 +556,15 @@ class BuilderLinux(BuilderBase):
             pass
         os.chdir(old_p)
 
-        utils.ensure_file_exists(os.path.join(self.PATH_RELEASE, 'lib', 'libssh.a'))
+        utils.ensure_file_exists(out_file)
-        files = os.listdir(os.path.join(self.PATH_RELEASE, 'lib'))
+        # files = os.listdir(os.path.join(self.PATH_RELEASE, 'lib'))
-        for i in files:
+        # for i in files:
-            if i.startswith('libssh.so'):
+        #     if i.startswith('libssh.so'):
-                # use os.unlink() because some file should be a link.
+        #         # use os.unlink() because some file should be a link.
-                os.unlink(os.path.join(self.PATH_RELEASE, 'lib', i))
+        #         os.unlink(os.path.join(self.PATH_RELEASE, 'lib', i))
+
+    def _build_zlib(self, file_name):
+        cc.w('skip build zlib again.')
 
     def fix_output(self):
         pass

build/builder/build-pysrt.py

@@ -250,7 +250,7 @@ class PYSLinux(PYSBase):
 
     def _get_pip(self):
         _exec_path = os.path.dirname(env.py_exec)
-        return os.path.join(_exec_path, 'pip')
+        return os.path.join(_exec_path, 'pip3.7')
 
     def _make_py_ver_file(self):
         # do nothing.

external/fix-external/libssh/libssh-0.9.2/src/libcrypto-compat.c

@@ -0,0 +1,398 @@
+/*
+ * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL license (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include "config.h"
+
+#include <string.h>
+#include "libcrypto-compat.h"
+
+#ifndef OPENSSL_NO_ENGINE
+#include <openssl/engine.h>
+#endif
+
+static void *OPENSSL_zalloc(size_t num)
+{
+    void *ret = OPENSSL_malloc(num);
+
+    if (ret != NULL)
+        memset(ret, 0, num);
+    return ret;
+}
+
+int RSA_set0_key(RSA *r, BIGNUM *n, BIGNUM *e, BIGNUM *d)
+{
+    /* If the fields n and e in r are NULL, the corresponding input
+     * parameters MUST be non-NULL for n and e.  d may be
+     * left NULL (in case only the public key is used).
+     */
+    if ((r->n == NULL && n == NULL)
+        || (r->e == NULL && e == NULL))
+        return 0;
+
+    if (n != NULL) {
+        BN_free(r->n);
+        r->n = n;
+    }
+    if (e != NULL) {
+        BN_free(r->e);
+        r->e = e;
+    }
+    if (d != NULL) {
+        BN_free(r->d);
+        r->d = d;
+    }
+
+    return 1;
+}
+
+int RSA_set0_factors(RSA *r, BIGNUM *p, BIGNUM *q)
+{
+    /* If the fields p and q in r are NULL, the corresponding input
+     * parameters MUST be non-NULL.
+     */
+    if ((r->p == NULL && p == NULL)
+        || (r->q == NULL && q == NULL))
+        return 0;
+
+    if (p != NULL) {
+        BN_free(r->p);
+        r->p = p;
+    }
+    if (q != NULL) {
+        BN_free(r->q);
+        r->q = q;
+    }
+
+    return 1;
+}
+
+int RSA_set0_crt_params(RSA *r, BIGNUM *dmp1, BIGNUM *dmq1, BIGNUM *iqmp)
+{
+    /* If the fields dmp1, dmq1 and iqmp in r are NULL, the corresponding input
+     * parameters MUST be non-NULL.
+     */
+    if ((r->dmp1 == NULL && dmp1 == NULL)
+        || (r->dmq1 == NULL && dmq1 == NULL)
+        || (r->iqmp == NULL && iqmp == NULL))
+        return 0;
+
+    if (dmp1 != NULL) {
+        BN_free(r->dmp1);
+        r->dmp1 = dmp1;
+    }
+    if (dmq1 != NULL) {
+        BN_free(r->dmq1);
+        r->dmq1 = dmq1;
+    }
+    if (iqmp != NULL) {
+        BN_free(r->iqmp);
+        r->iqmp = iqmp;
+    }
+
+    return 1;
+}
+
+void RSA_get0_key(const RSA *r,
+                  const BIGNUM **n, const BIGNUM **e, const BIGNUM **d)
+{
+    if (n != NULL)
+        *n = r->n;
+    if (e != NULL)
+        *e = r->e;
+    if (d != NULL)
+        *d = r->d;
+}
+
+void RSA_get0_factors(const RSA *r, const BIGNUM **p, const BIGNUM **q)
+{
+    if (p != NULL)
+        *p = r->p;
+    if (q != NULL)
+        *q = r->q;
+}
+
+void RSA_get0_crt_params(const RSA *r,
+                         const BIGNUM **dmp1, const BIGNUM **dmq1,
+                         const BIGNUM **iqmp)
+{
+    if (dmp1 != NULL)
+        *dmp1 = r->dmp1;
+    if (dmq1 != NULL)
+        *dmq1 = r->dmq1;
+    if (iqmp != NULL)
+        *iqmp = r->iqmp;
+}
+
+void DSA_get0_pqg(const DSA *d,
+                  const BIGNUM **p, const BIGNUM **q, const BIGNUM **g)
+{
+    if (p != NULL)
+        *p = d->p;
+    if (q != NULL)
+        *q = d->q;
+    if (g != NULL)
+        *g = d->g;
+}
+
+int DSA_set0_pqg(DSA *d, BIGNUM *p, BIGNUM *q, BIGNUM *g)
+{
+    /* If the fields p, q and g in d are NULL, the corresponding input
+     * parameters MUST be non-NULL.
+     */
+    if ((d->p == NULL && p == NULL)
+        || (d->q == NULL && q == NULL)
+        || (d->g == NULL && g == NULL))
+        return 0;
+
+    if (p != NULL) {
+        BN_free(d->p);
+        d->p = p;
+    }
+    if (q != NULL) {
+        BN_free(d->q);
+        d->q = q;
+    }
+    if (g != NULL) {
+        BN_free(d->g);
+        d->g = g;
+    }
+
+    return 1;
+}
+
+void DSA_get0_key(const DSA *d,
+                  const BIGNUM **pub_key, const BIGNUM **priv_key)
+{
+    if (pub_key != NULL)
+        *pub_key = d->pub_key;
+    if (priv_key != NULL)
+        *priv_key = d->priv_key;
+}
+
+int DSA_set0_key(DSA *d, BIGNUM *pub_key, BIGNUM *priv_key)
+{
+    /* If the field pub_key in d is NULL, the corresponding input
+     * parameters MUST be non-NULL.  The priv_key field may
+     * be left NULL.
+     */
+    if (d->pub_key == NULL && pub_key == NULL)
+        return 0;
+
+    if (pub_key != NULL) {
+        BN_free(d->pub_key);
+        d->pub_key = pub_key;
+    }
+    if (priv_key != NULL) {
+        BN_free(d->priv_key);
+        d->priv_key = priv_key;
+    }
+
+    return 1;
+}
+
+void DSA_SIG_get0(const DSA_SIG *sig, const BIGNUM **pr, const BIGNUM **ps)
+{
+    if (pr != NULL)
+        *pr = sig->r;
+    if (ps != NULL)
+        *ps = sig->s;
+}
+
+int DSA_SIG_set0(DSA_SIG *sig, BIGNUM *r, BIGNUM *s)
+{
+    if (r == NULL || s == NULL)
+        return 0;
+    BN_clear_free(sig->r);
+    BN_clear_free(sig->s);
+    sig->r = r;
+    sig->s = s;
+    return 1;
+}
+
+void ECDSA_SIG_get0(const ECDSA_SIG *sig, const BIGNUM **pr, const BIGNUM **ps)
+{
+    if (pr != NULL)
+        *pr = sig->r;
+    if (ps != NULL)
+        *ps = sig->s;
+}
+
+int ECDSA_SIG_set0(ECDSA_SIG *sig, BIGNUM *r, BIGNUM *s)
+{
+    if (r == NULL || s == NULL)
+        return 0;
+    BN_clear_free(sig->r);
+    BN_clear_free(sig->s);
+    sig->r = r;
+    sig->s = s;
+    return 1;
+}
+
+EVP_MD_CTX *EVP_MD_CTX_new(void)
+{
+    return OPENSSL_zalloc(sizeof(EVP_MD_CTX));
+}
+
+static void OPENSSL_clear_free(void *str, size_t num)
+{
+    if (str == NULL)
+        return;
+    if (num)
+        OPENSSL_cleanse(str, num);
+    OPENSSL_free(str);
+}
+
+/* This call frees resources associated with the context */
+int EVP_MD_CTX_reset(EVP_MD_CTX *ctx)
+{
+    if (ctx == NULL)
+        return 1;
+
+    /*
+     * Don't assume ctx->md_data was cleaned in EVP_Digest_Final, because
+     * sometimes only copies of the context are ever finalised.
+     */
+    if (ctx->digest && ctx->digest->cleanup
+        && !EVP_MD_CTX_test_flags(ctx, EVP_MD_CTX_FLAG_CLEANED))
+        ctx->digest->cleanup(ctx);
+    if (ctx->digest && ctx->digest->ctx_size && ctx->md_data
+        && !EVP_MD_CTX_test_flags(ctx, EVP_MD_CTX_FLAG_REUSE)) {
+        OPENSSL_clear_free(ctx->md_data, ctx->digest->ctx_size);
+    }
+    EVP_PKEY_CTX_free(ctx->pctx);
+#ifndef OPENSSL_NO_ENGINE
+    ENGINE_finish(ctx->engine);
+#endif
+    OPENSSL_cleanse(ctx, sizeof(*ctx));
+
+    return 1;
+}
+
+void EVP_MD_CTX_free(EVP_MD_CTX *ctx)
+{
+    EVP_MD_CTX_reset(ctx);
+    OPENSSL_free(ctx);
+}
+
+HMAC_CTX *HMAC_CTX_new(void)
+{
+    HMAC_CTX *ctx = OPENSSL_zalloc(sizeof(HMAC_CTX));
+
+    if (ctx != NULL) {
+        if (!HMAC_CTX_reset(ctx)) {
+            HMAC_CTX_free(ctx);
+            return NULL;
+        }
+    }
+    return ctx;
+}
+
+static void hmac_ctx_cleanup(HMAC_CTX *ctx)
+{
+    EVP_MD_CTX_reset(&ctx->i_ctx);
+    EVP_MD_CTX_reset(&ctx->o_ctx);
+    EVP_MD_CTX_reset(&ctx->md_ctx);
+    ctx->md = NULL;
+    ctx->key_length = 0;
+    OPENSSL_cleanse(ctx->key, sizeof(ctx->key));
+}
+
+void HMAC_CTX_free(HMAC_CTX *ctx)
+{
+    if (ctx != NULL) {
+        hmac_ctx_cleanup(ctx);
+#if OPENSSL_VERSION_NUMBER > 0x10100000L
+        EVP_MD_CTX_free(&ctx->i_ctx);
+        EVP_MD_CTX_free(&ctx->o_ctx);
+        EVP_MD_CTX_free(&ctx->md_ctx);
+#endif
+        OPENSSL_free(ctx);
+    }
+}
+
+int HMAC_CTX_reset(HMAC_CTX *ctx)
+{
+    HMAC_CTX_init(ctx);
+    return 1;
+}
+
+#if 0 // by Apex
+#ifndef HAVE_OPENSSL_EVP_CIPHER_CTX_NEW
+EVP_CIPHER_CTX *EVP_CIPHER_CTX_new(void)
+{
+    return OPENSSL_zalloc(sizeof(EVP_CIPHER_CTX));
+}
+
+void EVP_CIPHER_CTX_free(EVP_CIPHER_CTX *ctx)
+{
+    /* EVP_CIPHER_CTX_reset(ctx); alias */
+    EVP_CIPHER_CTX_init(ctx);
+    OPENSSL_free(ctx);
+}
+#endif
+#endif
+
+void DH_get0_pqg(const DH *dh,
+                 const BIGNUM **p, const BIGNUM **q, const BIGNUM **g)
+{
+    if (p) {
+        *p = dh->p;
+    }
+    if (q) {
+        *q = NULL;
+    }
+    if (g) {
+        *g = dh->g;
+    }
+}
+
+int DH_set0_pqg(DH *dh, BIGNUM *p, BIGNUM *q, BIGNUM *g)
+{
+    if (p) {
+        if (dh->p) {
+            BN_free(dh->p);
+        }
+        dh->p = p;
+    }
+    if (g) {
+        if (dh->g) {
+            BN_free(dh->g);
+        }
+        dh->g = g;
+    }
+    return 1;
+}
+
+void DH_get0_key(const DH *dh,
+                 const BIGNUM **pub_key, const BIGNUM **priv_key)
+{
+    if (pub_key) {
+        *pub_key = dh->pub_key;
+    }
+    if (priv_key) {
+        *priv_key = dh->priv_key;
+    }
+}
+
+int DH_set0_key(DH *dh, BIGNUM *pub_key, BIGNUM *priv_key)
+{
+    if (pub_key) {
+        if (dh->pub_key) {
+            BN_free(dh->pub_key);
+        }
+        dh->pub_key = pub_key;
+    }
+    if (priv_key) {
+        if (dh->priv_key) {
+            BN_free(dh->priv_key);
+        }
+        dh->priv_key = priv_key;
+    }
+    return 1;
+}