From 5471356326398d5b99851cd71d59f44446799f35 Mon Sep 17 00:00:00 2001 From: Mehmet Bektas Date: Sun, 21 Jan 2024 15:57:04 -0800 Subject: [PATCH 1/5] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 4cf36e89..7f0d5a3e 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ __pycache__ env_installer/jlab_server/ env_installer/jlab_server.tar.gz + From 65cc9a8e697e19ca1de1983d0b9c168fd5b1d4df Mon Sep 17 00:00:00 2001 From: Mehmet Bektas Date: Sun, 21 Jan 2024 16:54:21 -0800 Subject: [PATCH 2/5] sign libexec dir --- package.json | 2 +- scripts/buildutil.js | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 8aef8b24..47a5e828 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "dist:win-arm64": "yarn build && yarn electron-builder --arm64 --publish never", "update_workflow_conda_lock": "cd workflow_env && rimraf *.lock && conda-lock --kind explicit -f publish_env.yaml && cd -", "update_conda_lock": "cd env_installer && rimraf *.lock && conda-lock --kind explicit -f jlab_server.yaml && cd -", - "clean_env_installer": "rimraf ./env_installer/jlab_server.tar.gz && conda env remove -p ./env_installer/jlab_server -y", + "clean_env_installer": "rimraf ./env_installer/jlab_server.tar.gz && rimraf ./env_installer/jlab_server", "create_env_installer:linux": "yarn clean_env_installer && conda-lock install --prefix ./env_installer/jlab_server ./env_installer/conda-linux-64.lock && conda pack -p ./env_installer/jlab_server -o ./env_installer/jlab_server.tar.gz", "create_env_installer:osx-64": "yarn clean_env_installer && conda-lock install --prefix ./env_installer/jlab_server ./env_installer/conda-osx-64.lock && conda pack -p ./env_installer/jlab_server -o ./env_installer/jlab_server.tar.gz", "create_env_installer:osx-arm64": "yarn clean_env_installer && conda-lock install --no-validate-platform --prefix ./env_installer/jlab_server ./env_installer/conda-osx-arm64.lock && conda pack -p ./env_installer/jlab_server -o ./env_installer/jlab_server.tar.gz", diff --git a/scripts/buildutil.js b/scripts/buildutil.js index 391f2f27..adf8d67a 100644 --- a/scripts/buildutil.js +++ b/scripts/buildutil.js @@ -94,11 +94,13 @@ if (cli.flags.updateBinarySignList) { const { isBinary } = require('istextorbinary'); const envInstallerDir = path.resolve('env_installer', 'jlab_server'); const envBinDir = path.join(envInstallerDir, 'bin'); + const libExecDir = path.join(envInstallerDir, 'libexec'); const needsSigning = filePath => { - // conly consider bin directory, and .so, .dylib files in other directories + // only consider bin & libexec directory, and .so, .dylib files in other directories if ( filePath.startsWith(envBinDir) || + filePath.startsWith(libExecDir) || filePath.endsWith('.so') || filePath.endsWith('.dylib') ) { From b7c426b485a07da4020b260a7846494cce7d8286 Mon Sep 17 00:00:00 2001 From: Mehmet Bektas Date: Sun, 21 Jan 2024 17:00:05 -0800 Subject: [PATCH 3/5] update sign lists --- env_installer/sign-osx-64.txt | 4 ++++ env_installer/sign-osx-arm64.txt | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/env_installer/sign-osx-64.txt b/env_installer/sign-osx-64.txt index d027ce8b..3351da67 100644 --- a/env_installer/sign-osx-64.txt +++ b/env_installer/sign-osx-64.txt @@ -516,3 +516,7 @@ lib/tdbcmysql1.1.5/libtdbcmysql1.1.5.dylib lib/tdbcodbc1.1.5/libtdbcodbc1.1.5.dylib lib/tdbcpostgres1.1.5/libtdbcpostgres1.1.5.dylib lib/thread2.8.8/libthread2.8.8.dylib +libexec/lzo/examples/lzopack +libexec/lzo/examples/lzotest +libexec/lzo/examples/simple +libexec/lzo/examples/testmini diff --git a/env_installer/sign-osx-arm64.txt b/env_installer/sign-osx-arm64.txt index e75a7060..c7e30a06 100644 --- a/env_installer/sign-osx-arm64.txt +++ b/env_installer/sign-osx-arm64.txt @@ -508,3 +508,7 @@ lib/tdbcmysql1.1.5/libtdbcmysql1.1.5.dylib lib/tdbcodbc1.1.5/libtdbcodbc1.1.5.dylib lib/tdbcpostgres1.1.5/libtdbcpostgres1.1.5.dylib lib/thread2.8.8/libthread2.8.8.dylib +libexec/lzo/examples/lzopack +libexec/lzo/examples/lzotest +libexec/lzo/examples/simple +libexec/lzo/examples/testmini From cba03c805b927a20e28cba2cc1460ddc05b23dca Mon Sep 17 00:00:00 2001 From: Mehmet Bektas Date: Sun, 21 Jan 2024 17:18:11 -0800 Subject: [PATCH 4/5] add sbin dir to sign list --- env_installer/sign-osx-64.txt | 11 +++++++++++ env_installer/sign-osx-arm64.txt | 11 +++++++++++ scripts/buildutil.js | 8 +++++--- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/env_installer/sign-osx-64.txt b/env_installer/sign-osx-64.txt index 3351da67..4324f59b 100644 --- a/env_installer/sign-osx-64.txt +++ b/env_installer/sign-osx-64.txt @@ -520,3 +520,14 @@ libexec/lzo/examples/lzopack libexec/lzo/examples/lzotest libexec/lzo/examples/simple libexec/lzo/examples/testmini +sbin/gss-server +sbin/kadmin.local +sbin/kadmind +sbin/kdb5_util +sbin/kprop +sbin/kpropd +sbin/kproplog +sbin/krb5kdc +sbin/sim_server +sbin/sserver +sbin/uuserver diff --git a/env_installer/sign-osx-arm64.txt b/env_installer/sign-osx-arm64.txt index c7e30a06..b3fa1251 100644 --- a/env_installer/sign-osx-arm64.txt +++ b/env_installer/sign-osx-arm64.txt @@ -512,3 +512,14 @@ libexec/lzo/examples/lzopack libexec/lzo/examples/lzotest libexec/lzo/examples/simple libexec/lzo/examples/testmini +sbin/gss-server +sbin/kadmin.local +sbin/kadmind +sbin/kdb5_util +sbin/kprop +sbin/kpropd +sbin/kproplog +sbin/krb5kdc +sbin/sim_server +sbin/sserver +sbin/uuserver diff --git a/scripts/buildutil.js b/scripts/buildutil.js index adf8d67a..49a80483 100644 --- a/scripts/buildutil.js +++ b/scripts/buildutil.js @@ -94,13 +94,15 @@ if (cli.flags.updateBinarySignList) { const { isBinary } = require('istextorbinary'); const envInstallerDir = path.resolve('env_installer', 'jlab_server'); const envBinDir = path.join(envInstallerDir, 'bin'); - const libExecDir = path.join(envInstallerDir, 'libexec'); + const envSbinDir = path.join(envInstallerDir, 'sbin'); + const envLibexecDir = path.join(envInstallerDir, 'libexec'); const needsSigning = filePath => { - // only consider bin & libexec directory, and .so, .dylib files in other directories + // consider bin, libexec, sbin directories, and .so, .dylib files in other directories if ( filePath.startsWith(envBinDir) || - filePath.startsWith(libExecDir) || + filePath.startsWith(envLibexecDir) || + filePath.startsWith(envSbinDir) || filePath.endsWith('.so') || filePath.endsWith('.dylib') ) { From d34e54b5e7a3415e9c236435773b29c54bf3b9e1 Mon Sep 17 00:00:00 2001 From: Mehmet Bektas Date: Sun, 21 Jan 2024 22:26:55 -0800 Subject: [PATCH 5/5] fix code signing --- env_installer/sign-osx-64.txt | 18 +++++++++ env_installer/sign-osx-arm64.txt | 18 +++++++++ scripts/buildutil.js | 66 +++++++++++++++++++++++++------- 3 files changed, 88 insertions(+), 14 deletions(-) diff --git a/env_installer/sign-osx-64.txt b/env_installer/sign-osx-64.txt index 4324f59b..cc752c40 100644 --- a/env_installer/sign-osx-64.txt +++ b/env_installer/sign-osx-64.txt @@ -204,6 +204,7 @@ lib/libyaml-cpp.0.8.0.dylib lib/libz.1.2.13.dylib lib/libzmq.5.dylib lib/libzstd.1.5.5.dylib +lib/python3.8/config-3.8-darwin/python.o lib/python3.8/lib-dynload/_asyncio.cpython-38-darwin.so lib/python3.8/lib-dynload/_bisect.cpython-38-darwin.so lib/python3.8/lib-dynload/_blake2.cpython-38-darwin.so @@ -277,12 +278,15 @@ lib/python3.8/lib-dynload/termios.cpython-38-darwin.so lib/python3.8/lib-dynload/unicodedata.cpython-38-darwin.so lib/python3.8/lib-dynload/xxlimited.cpython-38-darwin.so lib/python3.8/lib-dynload/zlib.cpython-38-darwin.so +lib/python3.8/lib2to3/Grammar3.8.18.final.0.pickle +lib/python3.8/lib2to3/PatternGrammar3.8.18.final.0.pickle lib/python3.8/site-packages/AppKit/_AppKit.cpython-38-darwin.so lib/python3.8/site-packages/AppKit/_inlines.cpython-38-darwin.so lib/python3.8/site-packages/CoreFoundation/_CoreFoundation.cpython-38-darwin.so lib/python3.8/site-packages/CoreFoundation/_inlines.cpython-38-darwin.so lib/python3.8/site-packages/Foundation/_Foundation.cpython-38-darwin.so lib/python3.8/site-packages/Foundation/_inlines.cpython-38-darwin.so +lib/python3.8/site-packages/IPython/core/tests/nonascii.py lib/python3.8/site-packages/PIL/_imaging.cpython-38-darwin.so lib/python3.8/site-packages/PIL/_imagingcms.cpython-38-darwin.so lib/python3.8/site-packages/PIL/_imagingft.cpython-38-darwin.so @@ -298,12 +302,20 @@ lib/python3.8/site-packages/contourpy/_contourpy.cpython-38-darwin.so lib/python3.8/site-packages/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_cython.cpython-38-darwin.so lib/python3.8/site-packages/debugpy/_vendored/pydevd/_pydevd_frame_eval/pydevd_frame_evaluator.cpython-38-darwin.so lib/python3.8/site-packages/debugpy/_vendored/pydevd/pydevd_attach_to_process/attach_x86_64.dylib +lib/python3.8/site-packages/debugpy/_vendored/pydevd/tests_python/resources/_pydev_coverage_cyrillic_encoding_py2.py +lib/python3.8/site-packages/debugpy/_vendored/pydevd/tests_python/resources/_pydev_coverage_cyrillic_encoding_py3.py +lib/python3.8/site-packages/debugpy/_vendored/pydevd/tests_python/resources/_pydev_coverage_syntax_error.py lib/python3.8/site-packages/fontTools/cu2qu/cu2qu.cpython-38-darwin.so lib/python3.8/site-packages/fontTools/feaLib/lexer.cpython-38-darwin.so lib/python3.8/site-packages/fontTools/misc/bezierTools.cpython-38-darwin.so lib/python3.8/site-packages/fontTools/pens/momentsPen.cpython-38-darwin.so lib/python3.8/site-packages/fontTools/qu2cu/qu2cu.cpython-38-darwin.so lib/python3.8/site-packages/fontTools/varLib/iup.cpython-38-darwin.so +lib/python3.8/site-packages/importlib_resources/tests/data01/binary.file +lib/python3.8/site-packages/importlib_resources/tests/data01/subdirectory/binary.file +lib/python3.8/site-packages/importlib_resources/tests/data01/utf-16.file +lib/python3.8/site-packages/importlib_resources/tests/namespacedata01/binary.file +lib/python3.8/site-packages/importlib_resources/tests/namespacedata01/utf-16.file lib/python3.8/site-packages/kiwisolver/_cext.cpython-38-darwin.so lib/python3.8/site-packages/libmambapy/bindings.cpython-38-darwin.so lib/python3.8/site-packages/markupsafe/_speedups.cpython-38-darwin.so @@ -317,6 +329,10 @@ lib/python3.8/site-packages/matplotlib/backends/_backend_agg.cpython-38-darwin.s lib/python3.8/site-packages/matplotlib/backends/_macosx.cpython-38-darwin.so lib/python3.8/site-packages/matplotlib/backends/_tkagg.cpython-38-darwin.so lib/python3.8/site-packages/matplotlib/ft2font.cpython-38-darwin.so +lib/python3.8/site-packages/menuinst/data/appkit_launcher_arm64 +lib/python3.8/site-packages/menuinst/data/appkit_launcher_x86_64 +lib/python3.8/site-packages/menuinst/data/osx_launcher_arm64 +lib/python3.8/site-packages/menuinst/data/osx_launcher_x86_64 lib/python3.8/site-packages/numpy/core/_multiarray_tests.cpython-38-darwin.so lib/python3.8/site-packages/numpy/core/_multiarray_umath.cpython-38-darwin.so lib/python3.8/site-packages/numpy/core/_operand_flag_tests.cpython-38-darwin.so @@ -324,6 +340,8 @@ lib/python3.8/site-packages/numpy/core/_rational_tests.cpython-38-darwin.so lib/python3.8/site-packages/numpy/core/_simd.cpython-38-darwin.so lib/python3.8/site-packages/numpy/core/_struct_ufunc_tests.cpython-38-darwin.so lib/python3.8/site-packages/numpy/core/_umath_tests.cpython-38-darwin.so +lib/python3.8/site-packages/numpy/core/tests/data/recarray_from_file.fits +lib/python3.8/site-packages/numpy/f2py/tests/src/module_data/mod.mod lib/python3.8/site-packages/numpy/fft/_pocketfft_internal.cpython-38-darwin.so lib/python3.8/site-packages/numpy/linalg/_umath_linalg.cpython-38-darwin.so lib/python3.8/site-packages/numpy/linalg/lapack_lite.cpython-38-darwin.so diff --git a/env_installer/sign-osx-arm64.txt b/env_installer/sign-osx-arm64.txt index b3fa1251..02f83eca 100644 --- a/env_installer/sign-osx-arm64.txt +++ b/env_installer/sign-osx-arm64.txt @@ -204,6 +204,7 @@ lib/libyaml-cpp.0.8.0.dylib lib/libz.1.2.13.dylib lib/libzmq.5.dylib lib/libzstd.1.5.5.dylib +lib/python3.8/config-3.8-darwin/python.o lib/python3.8/lib-dynload/_asyncio.cpython-38-darwin.so lib/python3.8/lib-dynload/_bisect.cpython-38-darwin.so lib/python3.8/lib-dynload/_blake2.cpython-38-darwin.so @@ -277,6 +278,9 @@ lib/python3.8/lib-dynload/termios.cpython-38-darwin.so lib/python3.8/lib-dynload/unicodedata.cpython-38-darwin.so lib/python3.8/lib-dynload/xxlimited.cpython-38-darwin.so lib/python3.8/lib-dynload/zlib.cpython-38-darwin.so +lib/python3.8/lib2to3/Grammar3.8.18.final.0.pickle +lib/python3.8/lib2to3/PatternGrammar3.8.18.final.0.pickle +lib/python3.8/site-packages/IPython/core/tests/nonascii.py lib/python3.8/site-packages/PIL/_imaging.cpython-38-darwin.so lib/python3.8/site-packages/PIL/_imagingcms.cpython-38-darwin.so lib/python3.8/site-packages/PIL/_imagingft.cpython-38-darwin.so @@ -292,12 +296,20 @@ lib/python3.8/site-packages/contourpy/_contourpy.cpython-38-darwin.so lib/python3.8/site-packages/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_cython.cpython-38-darwin.so lib/python3.8/site-packages/debugpy/_vendored/pydevd/_pydevd_frame_eval/pydevd_frame_evaluator.cpython-38-darwin.so lib/python3.8/site-packages/debugpy/_vendored/pydevd/pydevd_attach_to_process/attach_linux_amd64.dylib +lib/python3.8/site-packages/debugpy/_vendored/pydevd/tests_python/resources/_pydev_coverage_cyrillic_encoding_py2.py +lib/python3.8/site-packages/debugpy/_vendored/pydevd/tests_python/resources/_pydev_coverage_cyrillic_encoding_py3.py +lib/python3.8/site-packages/debugpy/_vendored/pydevd/tests_python/resources/_pydev_coverage_syntax_error.py lib/python3.8/site-packages/fontTools/cu2qu/cu2qu.cpython-38-darwin.so lib/python3.8/site-packages/fontTools/feaLib/lexer.cpython-38-darwin.so lib/python3.8/site-packages/fontTools/misc/bezierTools.cpython-38-darwin.so lib/python3.8/site-packages/fontTools/pens/momentsPen.cpython-38-darwin.so lib/python3.8/site-packages/fontTools/qu2cu/qu2cu.cpython-38-darwin.so lib/python3.8/site-packages/fontTools/varLib/iup.cpython-38-darwin.so +lib/python3.8/site-packages/importlib_resources/tests/data01/binary.file +lib/python3.8/site-packages/importlib_resources/tests/data01/subdirectory/binary.file +lib/python3.8/site-packages/importlib_resources/tests/data01/utf-16.file +lib/python3.8/site-packages/importlib_resources/tests/namespacedata01/binary.file +lib/python3.8/site-packages/importlib_resources/tests/namespacedata01/utf-16.file lib/python3.8/site-packages/kiwisolver/_cext.cpython-38-darwin.so lib/python3.8/site-packages/libmambapy/bindings.cpython-38-darwin.so lib/python3.8/site-packages/markupsafe/_speedups.cpython-38-darwin.so @@ -311,6 +323,10 @@ lib/python3.8/site-packages/matplotlib/backends/_backend_agg.cpython-38-darwin.s lib/python3.8/site-packages/matplotlib/backends/_macosx.cpython-38-darwin.so lib/python3.8/site-packages/matplotlib/backends/_tkagg.cpython-38-darwin.so lib/python3.8/site-packages/matplotlib/ft2font.cpython-38-darwin.so +lib/python3.8/site-packages/menuinst/data/appkit_launcher_arm64 +lib/python3.8/site-packages/menuinst/data/appkit_launcher_x86_64 +lib/python3.8/site-packages/menuinst/data/osx_launcher_arm64 +lib/python3.8/site-packages/menuinst/data/osx_launcher_x86_64 lib/python3.8/site-packages/numpy/core/_multiarray_tests.cpython-38-darwin.so lib/python3.8/site-packages/numpy/core/_multiarray_umath.cpython-38-darwin.so lib/python3.8/site-packages/numpy/core/_operand_flag_tests.cpython-38-darwin.so @@ -318,6 +334,8 @@ lib/python3.8/site-packages/numpy/core/_rational_tests.cpython-38-darwin.so lib/python3.8/site-packages/numpy/core/_simd.cpython-38-darwin.so lib/python3.8/site-packages/numpy/core/_struct_ufunc_tests.cpython-38-darwin.so lib/python3.8/site-packages/numpy/core/_umath_tests.cpython-38-darwin.so +lib/python3.8/site-packages/numpy/core/tests/data/recarray_from_file.fits +lib/python3.8/site-packages/numpy/f2py/tests/src/module_data/mod.mod lib/python3.8/site-packages/numpy/fft/_pocketfft_internal.cpython-38-darwin.so lib/python3.8/site-packages/numpy/linalg/_umath_linalg.cpython-38-darwin.so lib/python3.8/site-packages/numpy/linalg/lapack_lite.cpython-38-darwin.so diff --git a/scripts/buildutil.js b/scripts/buildutil.js index 49a80483..6b1e6b3d 100644 --- a/scripts/buildutil.js +++ b/scripts/buildutil.js @@ -93,24 +93,62 @@ if (cli.flags.checkVersionMatch) { if (cli.flags.updateBinarySignList) { const { isBinary } = require('istextorbinary'); const envInstallerDir = path.resolve('env_installer', 'jlab_server'); - const envBinDir = path.join(envInstallerDir, 'bin'); - const envSbinDir = path.join(envInstallerDir, 'sbin'); - const envLibexecDir = path.join(envInstallerDir, 'libexec'); + const getFileExtension = filePath => { + const lastDot = filePath.lastIndexOf('.'); + if (lastDot !== -1) { + return filePath.substring(lastDot + 1); + } + }; + + const skipExtensions = new Set([ + 'a', + 'bz2', + 'dat', + 'eot', + 'exe', + 'gif', + 'gz', + 'jpg', + 'icns', + 'ico', + 'mo', + 'npy', + 'npz', + 'parquet', + 'pdf', + 'pkl', + 'png', + 'ppm', + 'pyc', + 'testcase', + 'tiff', + 'ttf', + 'wav', + 'whl', + 'woff', + 'woff2', + 'xz', + 'zip' + ]); + + const skipPathComponents = [ + '/pytz/zoneinfo/', + '/tzdata/zoneinfo/', + 'share/terminfo/' + ]; + + // sign binary files except for certain extensions and certain directories const needsSigning = filePath => { - // consider bin, libexec, sbin directories, and .so, .dylib files in other directories - if ( - filePath.startsWith(envBinDir) || - filePath.startsWith(envLibexecDir) || - filePath.startsWith(envSbinDir) || - filePath.endsWith('.so') || - filePath.endsWith('.dylib') - ) { - // check for binary content - return isBinary(null, fs.readFileSync(filePath)); + const skippedPath = skipPathComponents.find(component => { + return filePath.includes(component); + }); + + if (skippedPath || skipExtensions.has(getFileExtension(filePath))) { + return false; } - return false; + return isBinary(null, fs.readFileSync(filePath)); }; const findBinariesInDirectory = dirPath => {