`make_modules` doesn't scan local modules for dependencies

[CookiePLMonster]

At https://cookieplmonster.github.io/bonuscodes/toca-race-driver-3/ I have an inline Python script that imports a few more Python modules I have in my website's PYTHONPATH. Some of those modules in turn import more Brython and Python modules:

• htmlgen.py imports Template from browser.template.
• rd3.py imports struct and collections.namedtuple.

This setup works perfectly with brython_stdlib.js, with the exception of the rd2 module returning a single 404 (visible in the Console), but that is expected as per the way Brython works (or maybemake_modules is intended to solve this too?), but with python_modules.js generated stops working.

Running make_modules in the root directory of my blog produces this brython_modules.js file - as you can see, it's missing template, struct and namedtuple:

brython_modules.js

__BRYTHON__.VFS_timestamp = 1744651669804
if(typeof document !== 'undefined'){
    __BRYTHON__.brython_modules = $B.last(document.getElementsByTagName('script')).src
}
__BRYTHON__.use_VFS = true
var scripts = {"$timestamp": 1744651669803, "_ajax": [".js", "// ajax\n__BRYTHON__.imported._ajax = (function($B){\n\n\nvar $N = $B.builtins.None,\n    _b_ = $B.builtins\n\nvar add_to_res = function(res, key, val) {\n    if($B.$isinstance(val, _b_.list)){\n        for (j = 0; j < val.length; j++) {\n            add_to_res(res, key, val[j])\n        }\n    }else if (val instanceof File || val instanceof Blob){\n        res.append(key, val)\n    }else{\n        res.append(key, _b_.str.$factory(val))\n    }\n}\n\nfunction set_timeout(self, timeout){\n    if(timeout.seconds !== undefined){\n        self.js.$requestTimer = setTimeout(\n            function() {\n                self.js.abort()\n                if(timeout.func){\n                    timeout.func()\n                }\n            },\n            timeout.seconds * 1000)\n    }\n}\n\nfunction _read(req){\n    var xhr = req.js\n    if(xhr.responseType == \"json\"){\n        return $B.structuredclone2pyobj(xhr.response)\n    }\n    if(req.charset_user_defined){\n        // on blocking mode, xhr.response is a string\n        var bytes = []\n        for(var i = 0, len = xhr.response.length; i < len; i++){\n            var cp = xhr.response.codePointAt(i)\n            if(cp > 0xf700){\n                bytes.push(cp - 0xf700)\n            }else{\n                bytes.push(cp)\n            }\n        }\n    }else if(typeof xhr.response == \"string\"){\n        if(req.mode == 'binary'){\n            return _b_.str.encode(xhr.response,\n                $B.$getattr(req, 'encoding', 'utf-8'))\n        }\n        return xhr.response\n    }else{\n        // else it's an ArrayBuffer\n        var buf = new Uint8Array(xhr.response),\n            bytes = Array.from(buf.values())\n    }\n    var b = _b_.bytes.$factory(bytes),\n        mode = $B.$getattr(req, 'mode', null)\n    if(mode == \"binary\"){\n        return b\n    }else if(mode == \"document\"){\n        return $B.jsobj2pyobj(xhr.response)\n    }else{\n        var encoding = $B.$getattr(req, 'encoding', \"utf-8\")\n        return _b_.bytes.decode(b, encoding)\n    }\n}\n\nfunction stringify(d){\n    var items = []\n    for(var entry of _b_.dict.$iter_items(d)){\n        items.push(encodeURIComponent(entry.key) + \"=\" +\n                   encodeURIComponent(entry.value))\n    }\n    return items.join(\"&\")\n}\n\nfunction handle_kwargs(self, kw, method){\n    var data,\n        encoding,\n        headers = {},\n        cache,\n        mode = \"text\",\n        timeout = {},\n        rawdata\n\n    for(var item of _b_.dict.$iter_items(kw)){\n        var key = item.key\n        if(key == \"data\"){\n            var rawdata = item.value\n            if(typeof rawdata == \"string\" || rawdata instanceof FormData){\n                data = rawdata\n            }else if(rawdata.__class__ === _b_.dict){\n                data = stringify(rawdata)\n            }else{\n                throw _b_.TypeError.$factory(\"wrong type for data: \" +\n                    $B.class_name(rawdata))\n            }\n        }else if(key == \"encoding\"){\n            encoding = item.value\n        }else if(key == \"headers\"){\n            var value = item.value\n            if(! $B.$isinstance(value, _b_.dict)){\n                throw _b_.ValueError.$factory(\n                    \"headers must be a dict, not \" + $B.class_name(value))\n            }\n            for(var subitem of _b_.dict.$iter_items(value)){\n                headers[subitem.key.toLowerCase()] = subitem.value\n            }\n        }else if(key.startsWith(\"on\")){\n            var event = key.substr(2)\n            if(event == \"timeout\"){\n                timeout.func = item.value\n            }else{\n                var f = item.value\n                ajax.bind(self, event, f)\n            }\n        }else if(key == \"mode\"){\n            var mode = item.value\n        }else if(key == \"timeout\"){\n            timeout.seconds = item.value\n        }else if(key == \"cache\"){\n            cache = item.value\n        }\n    }\n    if(encoding && mode != \"text\"){\n        throw _b_.ValueError.$factory(\"encoding not supported for mode \" +\n            mode)\n    }\n    if((method == \"post\" || method == \"put\") && ! headers){\n        // For POST requests, set default header\n        self.js.setRequestHeader(\"Content-type\",\n                                 \"application/x-www-form-urlencoded\")\n    }\n\n    return {cache, data, rawdata, encoding, headers, mode, timeout}\n}\n\nvar ajax = $B.make_class('ajax')\n\najax.__repr__ = function(self){\n    return '<object Ajax>'\n}\n\najax.__getattribute__ = function(self, attr){\n    if(ajax[attr] !== undefined){\n        return function(){\n            return ajax[attr].call(null, self, ...arguments)\n        }\n    }else if(attr == \"text\"){\n        return _read(self)\n    }else if(attr == \"json\"){\n        if(self.js.responseType == \"json\"){\n            return _read(self)\n        }else{\n            var resp = _read(self)\n            try{\n                return $B.structuredclone2pyobj(JSON.parse(resp))\n            }catch(err){\n                console.log('attr json, invalid resp', resp)\n                throw err\n            }\n        }\n    }else if(self.js[attr] !== undefined){\n        if(typeof self.js[attr] == \"function\"){\n            return function(){\n                if(attr == \"setRequestHeader\"){\n                    ajax.set_header.call(null, self, ...arguments)\n                }else{\n                    if(attr == 'overrideMimeType'){\n                        console.log('override mime type')\n                        self.hasMimeType = true\n                    }\n                    return self.js[attr](...arguments)\n                }\n            }\n        }else{\n            return self.js[attr]\n        }\n    }else if(attr == \"xml\"){\n        return $B.jsobj2pyobj(self.js.responseXML)\n    }\n    return _b_.object.__getattribute__(self, attr)\n}\n\najax.bind = function(self, evt, func){\n    // req.bind(evt,func) is the same as req.onevt = func\n    self.js['on' + evt] = function(){\n        try{\n            return func.apply(null, arguments)\n        }catch(err){\n            $B.handle_error(err)\n        }\n    }\n    return _b_.None\n}\n\najax.open = function(){\n    var $ = $B.args('open', 4,\n            {self: null, method: null, url: null, async: null},\n            ['self', 'method', 'url', 'async'], arguments,\n            {async: true}, null, null),\n        self = $.self,\n        method = $.method,\n        url = $.url,\n        async = $.async\n    if(typeof method !== \"string\"){\n        throw _b_.TypeError.$factory(\n            'open() argument method should be string, got ' +\n            $B.class_name(method))\n    }\n    if(typeof url !== \"string\"){\n        throw _b_.TypeError.$factory(\n            'open() argument url should be string, got ' +\n            $B.class_name(url))\n    }\n    self.$method = method\n    self.blocking = ! self.async\n    self.js.open(method, url, async)\n}\n\najax.read = function(self){\n    return _read(self)\n}\n\najax.send = function(self, params){\n    // params can be Python dictionary or string\n    var content_type\n    for(var key in self.headers){\n        var value = self.headers[key]\n        self.js.setRequestHeader(key, value)\n        if(key == 'content-type'){\n            content_type = value\n        }\n    }\n    if(($B.$getattr(self, 'encoding', false) ||\n            $B.$getattr(self, 'blocking', false)) && ! self.hasMimeType){\n        // On blocking mode, or if an encoding has been specified,\n        // override Mime type so that bytes are not processed\n        // (unless the Mime type has been explicitely set)\n        self.js.overrideMimeType('text/plain;charset=x-user-defined')\n        self.charset_user_defined = true\n    }\n    var res = ''\n    if(! params){\n        self.js.send()\n        return _b_.None\n    }\n    if($B.$isinstance(params, _b_.str)){\n        res = params\n    }else if($B.$isinstance(params, _b_.dict)){\n        if(content_type == 'multipart/form-data'){\n            // The FormData object serializes the data in the 'multipart/form-data'\n            // content-type so we may as well override that header if it was set\n            // by the user.\n            res = new FormData()\n            var items = _b_.list.$factory(_b_.dict.items(params))\n            for(var i = 0, len = items.length; i < len; i++){\n                add_to_res(res, _b_.str.$factory(items[i][0]), items[i][1])\n            }\n        }else{\n            if(self.$method && self.$method.toUpperCase() == \"POST\" &&\n                    ! content_type){\n                // Set default Content-Type for POST requests\n                self.js.setRequestHeader(\"Content-Type\",\n                    \"application/x-www-form-urlencoded\")\n            }\n            var items = _b_.list.$factory(_b_.dict.items(params))\n            for(var i = 0, len = items.length; i < len; i++){\n                var key = encodeURIComponent(_b_.str.$factory(items[i][0]));\n                if($B.$isinstance(items[i][1], _b_.list)){\n                    for (j = 0; j < items[i][1].length; j++) {\n                        res += key +'=' +\n                            encodeURIComponent(_b_.str.$factory(items[i][1][j])) + '&'\n                    }\n                }else{\n                    res += key + '=' +\n                        encodeURIComponent(_b_.str.$factory(items[i][1])) + '&'\n                }\n            }\n            res = res.substr(0, res.length - 1)\n        }\n    }else if(params instanceof FormData){\n        res = params\n    }else{\n        throw _b_.TypeError.$factory(\n            \"send() argument must be string or dictionary, not '\" +\n            _b_.str.$factory(params.__class__) + \"'\")\n    }\n    self.js.send(res)\n    return _b_.None\n}\n\najax.responseType = _b_.property.$factory(\n    function(_self){\n        return _self.responseType\n    },\n    function(_self, value){\n        _self.js.responseType = value\n    }\n)\n\najax.withCredentials = _b_.property.$factory(\n    function(_self){\n        return _self.withCredentials\n    },\n    function(_self, value){\n        _self.js.withCredentials = value\n    }\n)\n\najax.set_header = function(self, key, value){\n    self.headers[key.toLowerCase()] = value\n}\n\najax.set_timeout = function(self, seconds, func){\n    self.js.$requestTimer = setTimeout(\n        function() {\n            self.js.abort()\n            func()\n        },\n        seconds * 1000)\n}\n\najax.$factory = function(){\n\n    var xmlhttp = new XMLHttpRequest()\n\n    xmlhttp.onreadystatechange = function(){\n        // here, \"this\" refers to xmlhttp\n        var state = this.readyState\n        if(this.responseType == \"\" || this.responseType == \"text\"){\n            res.js.text = this.responseText\n        }\n        var timer = this.$requestTimer\n        if(state == 0 && this.onuninitialized){\n            this.onuninitialized(res)\n        }else if(state == 1 && this.onloading){\n            this.onloading(res)\n        }else if(state == 2 && this.onloaded){\n            this.onloaded(res)\n        }else if(state == 3 && this.oninteractive){\n            this.oninteractive(res)\n        }else if(state == 4 && this.oncomplete){\n            if(timer !== null){\n                globalThis.clearTimeout(timer)\n            }\n            this.oncomplete(res)\n        }\n    }\n    var res = {\n        __class__: ajax,\n        __dict__: $B.empty_dict(),\n        js: xmlhttp,\n        headers: {}\n    }\n    return res\n}\n\n\nfunction _request_without_body(method){\n    var $ = $B.args(method, 3, {method: null, url: null, blocking: null},\n        [\"method\", \"url\", \"blocking\"], arguments, {blocking: false},\n        null, \"kw\"),\n    method = $.method,\n    url = $.url,\n    async = !$.blocking,\n    kw = $.kw\n\n    var self = ajax.$factory()\n    self.blocking = $.blocking\n    var items = handle_kwargs(self, kw, method),\n        mode = items.mode,\n        encoding = items.encoding,\n        qs = items.data\n    $B.$setattr(self, 'mode', mode)\n    $B.$setattr(self, 'encoding', encoding)\n    if(qs){\n        url += \"?\" + qs\n    }\n    if(! (items.cache === true)){\n        url += (qs ? \"&\" : \"?\") + (new Date()).getTime()\n    }\n    self.js.open(method.toUpperCase(), url, async)\n\n    if(async){\n        if(mode == \"json\" || mode == \"document\"){\n            self.js.responseType = mode\n        }else{\n            self.js.responseType = \"arraybuffer\"\n            if(mode != \"text\" && mode != \"binary\"){\n                throw _b_.ValueError.$factory(\"invalid mode: \" + mode)\n            }\n        }\n    }else{\n        self.js.overrideMimeType('text/plain;charset=x-user-defined')\n        self.charset_user_defined = true\n    }\n    for(var key in items.headers){\n        self.js.setRequestHeader(key, items.headers[key])\n    }\n    var timeout = items.timeout\n    if(timeout.seconds){\n        ajax.set_timeout(self, timeout.seconds, timeout.func)\n    }\n    // Add function read() to return str or bytes according to mode\n    self.js.send()\n}\n\nfunction _request_with_body(method){\n    var $ = $B.args(method, 3, {method: null, url: null, blocking: null},\n        [\"method\", \"url\", \"blocking\"], arguments, {blocking: false},\n        null, \"kw\"),\n        method = $.method,\n        url = $.url,\n        async = !$.blocking,\n        kw = $.kw,\n        content_type\n    var self = ajax.$factory()\n    self.js.open(method.toUpperCase(), url, async)\n    var items = handle_kwargs(self, kw, method), // common with browser.aio\n        data = items.data\n\n    if($B.$isinstance(data, _b_.dict)){\n        data = stringify(data)\n    }\n    for(var key in items.headers){\n        var value = items.headers[key]\n        self.js.setRequestHeader(key, value)\n        if(key == 'content-type'){\n            content_type = value\n        }\n    }\n    if(method.toUpperCase() == 'POST' && !content_type){\n        // set default Content-Type for POST requests\n        self.js.setRequestHeader('Content-Type',\n            'application/x-www-form-urlencoded')\n    }\n\n    // Add function read() to return str or bytes according to mode\n    self.js.read = function(){\n        return _read(self)\n    }\n    self.js.send(data)\n}\n\nfunction form_data(form){\n    var missing = {},\n        $ = $B.args('form_data', 1, {form: null}, ['form'], arguments,\n            {form: missing}, null, null)\n    if($.form === missing){\n        return new FormData()\n    }else{\n        return new FormData($.form)\n    }\n}\n\nfunction connect(){\n    _request_without_body.call(null, \"connect\", ...arguments)\n}\n\nfunction _delete(){\n    _request_without_body.call(null, \"delete\", ...arguments)\n}\n\nfunction get(){\n    _request_without_body.call(null, \"get\", ...arguments)\n}\n\nfunction head(){\n    _request_without_body.call(null, \"head\", ...arguments)\n}\n\nfunction options(){\n    _request_without_body.call(null, \"options\", ...arguments)\n}\n\nfunction patch(){\n    _request_with_body.call(null, \"put\", ...arguments)\n}\n\nfunction post(){\n    _request_with_body.call(null, \"post\", ...arguments)\n}\n\nfunction put(){\n    _request_with_body.call(null, \"put\", ...arguments)\n}\n\nfunction trace(){\n    _request_without_body.call(null, \"trace\", ...arguments)\n}\n\nfunction file_upload(){\n    // ajax.file_upload(url, file, method=\"POST\", **callbacks)\n    var $ = $B.args(\"file_upload\", 2, {url: null, \"file\": file},\n            [\"url\", \"file\"], arguments, {}, null, \"kw\"),\n        url = $.url,\n        file = $.file,\n        kw = $.kw\n\n    var self = ajax.$factory()\n\n    var items = handle_kwargs(self, kw, method),\n        rawdata = items.rawdata,\n        headers = items.headers\n\n    for(var key in headers){\n        var value = headers[key]\n        self.js.setRequestHeader(key, value)\n        if(key == 'content-type'){\n            content_type = value\n        }\n    }\n\n    var timeout = items.timeout\n    if(timeout.seconds){\n        ajax.set_timeout(self, timeout.seconds, timeout.func)\n    }\n\n    var method = _b_.dict.$get_string(kw, 'method', 'POST'),\n        field_name = _b_.dict.$get_string(kw, 'field_name', 'filetosave')\n\n    var formdata = new FormData()\n    formdata.append(field_name, file, file.name)\n\n    if(rawdata){\n        if(rawdata instanceof FormData){\n            // append additional data\n            for(var d of rawdata){\n                formdata.append(d[0], d[1])\n            }\n        }else if($B.$isinstance(rawdata, _b_.dict)){\n            for(var item of _b_.dict.$iter_items(rawdata)){\n                formdata.append(item.key, item.value)\n            }\n        }else{\n            throw _b_.ValueError.$factory(\n                'data value must be a dict of form_data')\n        }\n    }\n\n    self.js.open(method, url, _b_.True)\n    self.js.send(formdata)\n\n}\n\n$B.set_func_names(ajax)\n\nreturn {\n    ajax: ajax,\n    Ajax: ajax,\n    delete: _delete,\n    file_upload: file_upload,\n    connect,\n    form_data,\n    get,\n    head,\n    options,\n    patch,\n    post,\n    put,\n    trace\n}\n\n})(__BRYTHON__)\n"], "browser": [".py", "", [], 1], "browser.ajax": [".py", "from _ajax import *\n", ["_ajax"]]}
__BRYTHON__.update_VFS(scripts)

If I explicitly import any of those modules from one of my HTML files, make_modules gathers the dependency correctly.

My modules directory, added to PYTHONPATH with <brython-options pythonpath="/assets/brython">, is here - it exists on the deployed site, but isn't considered for the module scan:
https://github.com/CookiePLMonster/CookiePLMonster.github.io/tree/master/assets/brython

EDIT:
Perhaps alongside .py files, .bry files should also be scanned for dependencies, considering this specific "fake" extension is mentioned in the documentation?

[CookiePLMonster]

Oddly enough, the example from the documentation has a local module app, so I am not sure why would my setup not gather imports from the .py files?

[CookiePLMonster]

According to this https://github.com/brython-dev/brython/blob/master/setup/brython/list_modules.py#L569 there should be a scan for the user modules, I'll clone the script locally and debug it with my application. I can't see anything wrong at the first glance but maybe there's something (maybe some Windows specific error?).

EDIT:
If I add __init__py to my /assets/brython/ directory, those modules get scanned, but:

1. The resulting brython_modules.js is 2.5MB.
2. Searching for my module names like cmr3 or htmlgen in that file yields nothing.

[CookiePLMonster]

This might require a separate ticket, but I also found that --src-dir in make_package doesn't work. I specify make_package --src-dir assets\brython, yet directories from the current directory are still scanned for modules:

PS E:\CookiePLMonster.github.io> brython-cli make_package --src-dir assets\brython modules
Generating package modules
adding modules.brython package False
adding assets.gulpfile package False
adding assets.brython.htmlgen package False
adding assets.brython.generators.cmr04 package False
adding assets.brython.generators.cmr2005 package False
adding assets.brython.generators.cmr3 package False
adding assets.brython.generators.rd1 package False
adding assets.brython.generators.rd3 package False
adding assets.brython.generators package True
adding assets.brython.generators.rd2.base package False
adding assets.brython.generators.rd2.ps2 package False
adding assets.brython.generators.rd2 package True
adding assets.img.make_thumbs package False
adding assets.js.disqus.min package False

...and more...

I'm reporting it here because I wanted to settle on generating a package out of my .py files as a half-measure if I can't get make_modules to work.

Fixed in #2571.

[PierreQuentel]

I'm afraid I can't see a robust solution for this issue. The value for the attribute pythonpath is a list of urls, and the mapping between a url and a physical location on disk is determined by the server configuration. make_modules can get the urls, but has no way to know the matching disk location.

[CookiePLMonster]

I'm afraid I can't see a robust solution for this issue. The value for the attribute pythonpath is a list of urls, and the mapping between a url and a physical location on disk is determined by the server configuration. make_modules can get the urls, but has no way to know the matching disk location.

Is there any way I can change the website's directory structure to make make_modules work as intended? Or maybe this command could accept an optional pythonpath argument (using on-disk paths) to help steer it in the right direction?

EDIT:
Just to be clear on my intentions - I wish to:

1. Use pythonpath ,.py files and "upstream" stdlib during local development - like I do right now.
2. When deploying to production, have the CI install Brython and generate optimized imports, and use that optimized .js file instead of loose Python modules and upstream stdlib.

[CookiePLMonster]

As a side note - packaging my pythonpath into a .js file significantly reduced the performance issues of my web app, Interaction to Next Paint no longer takes over 0.5s on mobile devices.

If I manage to generate an optimized stdlib that includes all my dependencies, I suspect even the 0.2s INP warning will disappear.

[denis-migdal]

You can try :
1/ To split the js files into several chunk to load them in parallel.
2/ Try to lazy load some parts (but may be quite hard to do).
3/ Remove parts of the stdlib you aren't using.

[CookiePLMonster]

3/ Remove parts of the stdlib you aren't using.

Technically, that's exactly what make_modules is supposed to do.

[PierreQuentel]

In the latest commit referenced above I have added a command-line option brython-cli make_modules --modules_paths. The value is the location of a text file with linefeed-separated list of directories where to search Python modules and add them to brython_modules.js.

You can test it by generating a local version of brython-cli : go to folder /setup of the Brython repo, run python -m build, then pip install dist/brython-3.13.1-py3-none-any.whl (maybe adapted if you're not on Windows)

[CookiePLMonster]

Thank you! I will try it soon and report back.

[CookiePLMonster]

Hello @PierreQuentel, regarding this feature, are the modules paths meant to be relative to the current directory? If so, I have created a file modules_paths.txt in the root of my app, with the following contents:

assets\brython

and then in the same directory I run make_modules --modules_paths "modules_paths.txt", but the end result is still No change: brython_modules.js not updated, I can't see any of my modules (or their dependencies) in the generated JS file either. Am I setting it up wrong?

You can see the contents of my assets\brython here. When used with make_package or as a pythonpath, this setup works as-is.
https://github.com/CookiePLMonster/CookiePLMonster.github.io/tree/master/assets/brython

EDIT:
I know I am running a correct build of brython because of the --help output:

usage: brython-cli make_modules [-h] [-o OUTPUT_PATH] [--reset] [--modules_paths MODULES_PATHS]

Create brython_modules.js with all the modules used by the application

options:
  -h, --help            show this help message and exit
  -o, --output-path OUTPUT_PATH
                        Optionally specify modules file output path
  --reset               Reset brython_modules.js to stdlib
  --modules_paths MODULES_PATHS
                        Location of file with linefeed-separated list of paths

[CookiePLMonster]

@PierreQuentel Do you have any additional guidance on this issue? Anything I could check to help figure out if it's an error in the implementation or in how I use it?