uploading a large amount of jsonl files makes the upload extremely slow #37
Description
Hi there,
when trying to upload a lot (10000+ files) of jsonl files the whole process is starting out quite fast, and then after a couple houndred files slows down extremely so that it basically does not continue uploading.
i am trying to upload the files like this:
cjdb import -H localhost -p 5432 -U postgres -d testcjdb -s cjdb -f ./src/cjdb/ --srid "25832" --overwrite --transform --ignore-repeated-file
Any idea where i could look for the bottleneck?
the files i am trying to upload all look like this more or less:
{"type":"CityJSON","version":"2.0","CityObjects":{},"vertices":[],"transform":{"scale":[0.001,0.001,0.001],"translate":[608725.976,5744461.815,504.919]},"metadata":{"geographicalExtent":[608000.0,5744000.0,504.919,610000.0,5746000.0,560.919]}}
{"type":"CityJSONFeature","CityObjects":{"DEST_DESTGEOTR100bF4N":{"type":"Building","attributes":{"name":"Eckertalsperre","creationDate":"2023-03-22T00:00:00+01:00","Gemeindeschluessel":"15085190","DatenquelleDachhoehe":"1000","Geometrietyp2DReferenz":"3000","DatenquelleBodenhoehe":"1100","DatenquelleLage":"3000","Grundrissaktualitaet":"2022-11-18","measuredHeight":56.0,"function":"53009_2030"},"geometry":[{"type":"Solid","lod":"2","boundaries":[[[[0,1,2]],[[3,4,5]],[[6,2,7]],[[8,9,10]],[[11,12,13]],[[7,1,14]],[[12,1,13]],[[3,5,15]],[[3,15,8]],[[3,8,10]],[[1,16,13]],[[14,1,17]],[[11,18,12]],[[17,1,12]],[[13,16,19]],[[2,1,7]],[[20,21,0]],[[15,5,22]],[[4,13,19]],[[23,24,18]],[[18,24,17]],[[23,3,16]],[[3,11,4]],[[6,20,2]],[[23,16,1]],[[25,7,14]],[[4,11,13]],[[19,16,22]],[[3,10,16]],[[23,20,6]],[[21,1,0]],[[21,23,1]],[[8,15,9]],[[18,17,12]],[[23,6,26]],[[15,22,9]],[[22,16,9]],[[25,6,7]],[[23,21,20]],[[23,26,24]],[[24,25,14]],[[24,14,17]],[[3,23,11]],[[9,16,10]],[[20,0,2]],[[5,4,19]],[[5,19,22]],[[23,18,11]]]],"semantics":{"surfaces":[{"type":"GroundSurface","id":"ID_a39d7921-b819-4f23-b8a3-63e4cdae09f3"},{"type":"RoofSurface","id":"ID_1a422a35-c29a-47f7-82c1-0f84d0e0a0b4"},{"type":"WallSurface","id":"ID_c89e9008-add9-4c6a-b733-edbebfd5f16b"}],"values":[[0,1,2,2,2,0,0,1,1,2,0,0,2,0,0,0,2,2,2,1,2,2,1,2,2,2,2,0,2,1,2,2,2,2,1,2,0,2,1,1,2,2,1,0,2,2,2,1]]}}],"address":[{"Country":"Deutschland","Locality":"Ilsenburg (Harz)"}]}},"vertices":[[117788,137746,0],[120699,135153,0],[113396,132877,0],[71322,74379,56000],[12105,0,56000],[62432,63028,56000],[45721,99831,56000],[45721,99831,0],[78783,68029,56000],[69814,56996,0],[78783,68029,0],[0,8930,56000],[22848,59085,0],[0,8930,0],[21326,70660,0],[69814,56996,56000],[71322,74379,0],[23774,66758,0],[22848,59085,56000],[12105,0,0],[113396,132877,56000],[117788,137746,56000],[62432,63028,0],[120699,135153,56000],[23774,66758,56000],[21326,70660,56000],[21327,70660,56000]],"id":"DEST_DESTGEOTR100bF4N"}
same happens for not to be transformed files like this: (here without the --trynsform argument)
{"type":"CityJSON","version":"2.0","CityObjects":{},"vertices":[],"transform":{"scale":[0.001,0.001,0.001],"translate":[1262392.4107433285,6718324.982348329,362.757]},"metadata":{"geographicalExtent":[1262392.4107433285,6718324.982348329,362.757,1262858.1013059968,6719618.535790181,379.72],"referenceSystem":"https://www.opengis.net/def/crs/EPSG/0/3857"}}
{"type":"CityJSONFeature","CityObjects":{"DEST_DESTLIKA0001UzwO":{"type":"Building","attributes":{"creationDate":"2019-04-08T00:00:00+02:00","Gemeindeschluessel":"15087370","DatenquelleDachhoehe":"1000","Geometrietyp2DReferenz":"3000","DatenquelleBodenhoehe":"1200","DatenquelleLage":"1000","Grundrissaktualitaet":"2018-06-19","measuredHeight":3.746,"function":"31001_3060","roofType":"2100"},"geometry":[{"type":"Solid","lod":"2","boundaries":[[[[0,1,2,3]],[[1,4,5,2]],[[0,6,4,1]],[[6,0,3,7]],[[4,6,7,5]],[[2,5,7,3]]]],"semantics":{"surfaces":[{"type":"WallSurface","id":"UUID_635725c7-3d79-4266-8e83-0168f1c8b31b"},{"type":"WallSurface","id":"UUID_9bf0dda9-60d9-4647-9992-424a6fa779e7"},{"type":"RoofSurface","id":"UUID_32bbc7e9-2a1f-4263-b33b-123c4d538a32"},{"type":"WallSurface","id":"UUID_a0c688f7-f7d6-4355-a827-bd0e12695e95"},{"type":"WallSurface","id":"UUID_211455c7-cf51-4509-901f-2eace7feeb76"},{"type":"GroundSurface","id":"UUID_a10b994a-30e6-42ce-bfcb-d8587b8c197e"}],"values":[[0,1,2,3,4,5]]}}],"address":[{"Country":"Deutschland","Locality":"Sangerhausen, Stadt"}]}},"vertices":[[274352,13587,11033],[258591,21942,11033],[258591,21942,7287],[274352,13587,7287],[260276,25308,10383],[260276,25308,7287],[275742,16454,10483],[275742,16454,7287]],"id":"DEST_DESTLIKA0001UzwO"}
thanks a lot for ideas!