From 62de779a9fd5472979f476a2d39dcf5bea9b3ae8 Mon Sep 17 00:00:00 2001 From: Dingyuan Wang Date: Mon, 15 May 2023 10:23:30 +0800 Subject: [PATCH] write geotiff natively, remove gdal dependency --- README.md | 6 +- build.bat | 1 + icon.ico | Bin 0 -> 311454 bytes requirements.txt | 2 - tms2geotiff.py | 168 ++++++++++++++++++++++++++++++++++++----------- 5 files changed, 135 insertions(+), 42 deletions(-) create mode 100644 build.bat create mode 100644 icon.ico diff --git a/README.md b/README.md index b8fef4d..c6e37aa 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,12 @@ # tms2geotiff Download tiles from [Tile Map Server](https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames) (online maps) and make a large image. -If GDAL is installed, it can write to a GeoTIFF image. +If output is TIFF, it can write to a GeoTIFF image. Otherwise, it will save to a normal image with a World File for georeferencing (in EPSG:3857). -Dependencies: Pillow, numpy, requests/httpx, GDAL (optional) +Dependencies: Pillow, requests/httpx. + +The GDAL and numpy are no longer needed for writing GeoTIFF images. **GUI**: Directly run `python3 tms2geotiff.py` to open a GUI window. diff --git a/build.bat b/build.bat new file mode 100644 index 0000000..67fa9dd --- /dev/null +++ b/build.bat @@ -0,0 +1 @@ +python -mnuitka --standalone --python-flag=no_site --lto=yes --mingw64 --show-progress --nofollow-import-to=numpy --nofollow-import-to=osgeo --nofollow-import-to=requests --include-package=httpx --enable-plugin=tk-inter --disable-console --windows-icon-from-ico=icon.ico -j4 tms2geotiff.py diff --git a/icon.ico b/icon.ico new file mode 100644 index 0000000000000000000000000000000000000000..9e21daf03cffa93a230f2ef198c9693fa1b1149f GIT binary patch literal 311454 zcmeI5dyo~?oyV_R?&H7B{;`irWvi@>mC6Q9bhBnnAUZP)hz|&4Md1zzKA86i5$4WF zj6i@m0)~KtMq?1&jReK1xVwTIT?PX|<6|Z2#zhP)kbt88v9*;-?N+MJ_8B}w-#&eN zetoC!%<1mWRGsSEeNUf0{k^}>?|b?&-G#zAh2JlH>|+IQdS0RR+(O~HLZNWpd1vN7 z{U?RO9&g!27k%{orwWCWe_ANi)qV7RW1%o(lDC5Qr`P{<-MNL*%0l7O-n`eOx8%%n zZ&E0D?Tp9Y{`B>1?KSO^y z@nrO`yI8N{lq6-)+ci1tSN_?V9gvvHiipK`Y)(+?(m+k3T-dYOS^(_1o+3 zYQ6nKhYqRz`}c>IBS(%zZr`6%!anT{$>7c`ak=(zjf1J2x^|LUx8(W}OZVLR zZ9(tR^ZspxeGYCjY(8wg;Igo7vwidrizolGU>t?b`|DTwTq*u4)i=w-wijGx>PAi3 zez*crn(6O?T8GV#)Q0{(eCvh6Nb81eFSu-^HrRe;{VV-0EJuM_DWK1jc&Bg+3%xRY zp5%OQhQC}Yu)H|sB0ZirrTwOPDo!!$>1ki{>^~lB5%qS>F}9w5@WBV_)TvWtrP8s9 zUhZF4w>94V*XQT;HH!76n}&+MR`LG(?^is|m-b)c)4@7HuMe9K>aY8+uaW5Gy69^q z^nc{BR#1CAAEtM3Sr+v_^3L1p&@bK?D#!loXsY7p(MKOuaf)mF`uh|2vyR0|`7^7!yY`I`s-`kQR9TD z?@SE2(Eo!64@O>(@t?2P=g0kHAQ$?t`xCrJq0+qhO2I)3HT~D|@7Fl$^Usuk-lX8YN@4Shy>;}uYZhIpS^AGk;~z^wBcIlqe_;Ih_=Tz#uULNe5WZv{G&Q#KFuufFJ&3D0dB=d3l zZR$$e4c0MzjG6}5k<6Pu#hFUl4c3Y3xA`u(j$}SgzfE09yTLl9k5SX$I+A(Qr#Mqd zyTLk9{WjkP*OAP}>9?sXX*XEM^f780Tt_l*`V?m>X*XCWs^8|jIDIg6gWHkJn|0$% zO&_DCaoRI=CH)Q7G3&;eM)li#7pFZ_H`q7HyjeHS)bufG8mB!|SJK~L9kXtnX;i<> zcX8S?b%TAA%$s%NOidr7rg7RcbtU}`)-mhGnMU>7d>5xZQ#aT*$-G%N&eZfVY8t0K zQ&-a8U>&n=oM}|Q&3AFyGj)S~lgyiS<4jE-qo#4%Gj%2X4c0O1#+gR-+k6+NJySQ> zH_5zNH_p`bF=`s8JyTcG-(VfHZk%aUzs+}X+B0>7eUr?ab>mD;ADN;63ZMWAj9vkK zpIMz(R(fUlJ~O?5KNN6S0q+KebGnNS^~;Kr8Y=v2v2e=JwMK6n11{>Y?E3monrorvdery6*{${`&U;l3Y zqIIa%{(HwRKZ|qx5>|Is-RDk8j+acGe|>nZV*e}0zyG(_pZWcX|2*;U&->*d4kF$?a zb(eLecmMSmr_%o}L@n>{?;qAL&HI1s*fH_{6ZrqPK!4WfO>|tOdHnHf0#E`>%V2|9T{PoFy@$Uci=RE&szoylGCjXKDBGR4pl^18^Kk^^>e-ts0 zX8)Tp=`&Bxm^k6^Xv*yRi+^_F#eu`4Eu-m!S$F!R3qETlIToM*3ZMWApa2S>Ku#6V z&j}fN^6>BZo)c0yH>YZ_+bDnnD1ZVefC4Ch0w{n2D1ZVefC4Ch0w{n2D1ZVefC4Ch z0w{n2D1ZVefC4Ch0w{n2D1ZVefC4Ch0zm}^SFAXn+wdcNZt$;l1OlkCKcA z-Sg`1>X~1+w0B-zQ*ZI=zY1#4JPM4L0;Q(LOH1<_9$Qwd|DPnD<4GIk??bDg$MI5A z-Rc!{W_*0Ses{K;Zv}2!IQ1`kn(B5{h|iio@0aZk<%$38X_~z9@!7NgAm7o*-jAgM zy-ic5di?$OXc$~F|7z8C%L3JU%Z<4%Z8y(XOPkKT!?#*JOT~tNU(q`2&&Dza*j7Fj zSb9Ulm0q0wwo|0m`u!il_aUWyzSPGhTrKJ_MW#aE_?972S>%f)_K>4MM{so{J+Vv zriP!^G7e}vp9(CSJNY($oRz#|(Q16(p#lHgx! z`N%hlW&G>kXvKmn&dX<%ve&gzpl5EwgMR+kd&i)L=_o`K$Zlc&`x4rn)@3UoKs@AmV5fA_VC@_+jD>6)`|a*GRNfTa9i=iWPSoLV*p z(Dz{GGfLU(S}D+7Z1^`n{|}UAC(8eO@4W~A9?R8?rB>d2MH&CzJwaWyiUZosrvmW* zVWP(kN%}wh=ab2?-0Sh+|E)J(Q;%+5U3D4Q)*m{i$lCZH{>PGpd~Rzz_x(-{zrfRzsU#i5C643o?!exy77MS zo@2)U?lYU@7y|#t|NS`t{^7sYo}Fa-_&@$Hz5d7fKRdVK|M)-tZ*!647~AEx;2-|s zAO4dZlaFOvjQ<(`GybnWf8qc5KmK1mKQ52$C4GZ`_=kV^AImX&lG|eZ&-kD5fA#qb z|HuFF|LXaXjImv=1OM<3|L~vWn0zeTV*JnepYeb7`3wKY|MCCo`Ehw{FXQ`Ts3j zwy1%Dfn1eUt5&Jz=H{WY{`R)2-ur2erFj0Q`?&=Fuk8PZgS<}R-+g|=#`JNP!9V<0 z%5VL5w&VX3vz-IrznuTR-d43@DP^6f!=C}T@dzvP9C3rh;?54Q?*Um>A|L2#_T8;ZnyZFD& zTbyfiV^J6W;otC@^c4Q#pYeZw^9%gLKm5Z#$N%o;0mlFN?YqAoz!Wm-)ZrYpp}R z-`d)$cJAD%_U_$F31~1lIH)#l+B7^qdT^EenSk&Q|E}`C@4hb8-g?ISH*DD8#Je}f z>zyAXZCAgaprz%^F;ZWz{F&hJ5C6{c@14&wW8Tf1H^aa8?vJQEVEA9%-6}o@0RQk0 z|Iu7UG3U~E@b5m$=ANE7)4bmQr>|$>|MJ`f|L|XHFR!NkYTy3_|8n;AI0XLTzZyPk zy$<}x{0RPWs4uJn?{-1j4sZ6tP@|foP@ZrN1a{&Cqe>LalYQ0Xa&;Nb@``^z6 z|2hx6@WKo5?~RFFJyx$pzwrNP|KGD`PcHcH?Cex0PMm;$_*b>|>?GT-wg2m5;_chF ztB#J2%;teosTAq`od3)2m;nEe_J4CuOkV@|)vtb)xpMO4$%>wD_8tD=KS|F{vaPZ3 ze?JF#{ml8lY~}#`U*`XL<=*T!{KJ2ezMWKC`2WcF?iq$hGlhTnPm0+j+kt=h=l=id za}(qLk;nf>jvP@>Jn@8j=%I%)R~~-&VYO@5E~W2tj2a){|J5^Ai|h7DwF&>x{Ofz; zmn>P5*=v91{=aqW)+v3Bqf!olfA~*w+@4fhwI2UJ_Sj>YXW!%i{Xb6dpH0pyjTi9$ zX#d}~ZCh^mf8~`|GR^-l2Y>jT!+W>?hH_SegS#Gnb$0#5ze#K9y7tSzGw+wy(+5hk zMbD>|KL6|W*I!rq{Q$-zvpQYAe0k(KfGmvvPaGKNPiowmm3H3R^N$BBjSbSUyyL4A zRW13~<6-TK`W+anR;|ihS+i!%&^gVcM~_zYeysug52Wh8cxiru|LF05rQ_keEr);j zcX2#07XF_%*2DG}{^1|~$L3y}F#g7S8T`XP{KJ2m-~Yw=KRbtU{Lk^fWNd-|+w4fs zvAJ9i{zv9tKlfLkC-3d;&0M+t_S@CQjT_aWLx(v3|3WqykIFyrAI<;EFTXr|?JKL- z{`G!!cXxCBAO2JK?8920|KGZGYv%9$F?sE!mtLwk24MW3&9MRgAMO7;cI?Ov|8KnU z2K?u23;_Sp{2x1ZOzG!i8jsBCw7%Hrdzb1@ZrA|Cd}F%R?RfKbtuK{^4Jezu+JK^E&>Q@R9X8@DKm+&;7sL|C_Bm z!1$l>zhvx!|7VMPb5CH}`EL(A|Kq#Mu1)m$KN2?bP^TLH@4Rtp*|S-Ci}lUYXLZ-2 z9{>Mi=diT%zi9fUWihtCa|Zl-*XxtcNpblf?C(&eruwqqrgz@tX|>>5+9v#GkN*W% zUQ!lgJxvYj#~WKl%mMrE>r$;P%|rL~=>LC7nwKoKclz|{;roBqmRd*DcK`id>XC00 z%YGOAjJ6dEt~gK1YAx!)fA;vFb=iehoZa1awR(O_$B1#-yZFp=J-fQPhTeaD`0!yx zDc9ibx8GL&d!X-M**>DKyY5+}ZlB*6_+9D_)Pm_aZNh){_}AAYT4#PK5Nq|Suk~v6 z?02HQ>Y;D8Xa2t4B=1*lZ);Jzztb7X|G?UX>gx-pSieW_yt<}1tk$9~{AZK@S(jg^ zZktnQ#hmsa{XFnp8x|=&-q+(&f6>eCS>EQ2ZOdRmL>j@Ikcx7xm9>2`P9HEQ*uD^<_DhEVRc_N8LOf3BE))nCNn zv{rTDKfC_rRpS4~G{`14X$ph0TO;Fc0T%_ht`&+f}sxPQpuDMh#nlWIABhBphwfBat>oA3|+N%^bm?co1*ti!*pi8Ob*S`Yun|D~}B|M2g+Zzt71{NIjs__sBY z=1y1Z;s5x*G&bQM{*&@o*W1DW?O2C@TN7#SbhRG-kN-B{^fB3&0>+o-D zBF&wy*2Dkte`##OKl~@6RGPFL&U|M+o-DBF&wy*2Dkte`##OKm6nW z*^2@A&+l)F|KtDg5C3`bb?JU`{BJYC_}|tux>^ta$N#0V3IFh)c3+qFAOE*w9sX@iq`A}8diX#7FO5z3hkvPmroA5iZ^t_P z+nPvor>phwfBat>oA3|+Y4>$$|M7o2*5TjQM4CHYt%v{P|I*lmfB2XBXWHxG|8}gy zzpaThce+{+|HuEOu?he1pLSoD_8Hm-Zk3w__dtZB3-P)75(TKmIR`P56g@seh)u9{z8~ zI{e$3NOPyF_3(fEUmBb65C3WRb!q?ce>>LU-_}H$J6)}Z|KtDC*o1%hm-=Vg>*4=) zti!*pi8Ob*S`Yun|D~}B|L~u7UzheD|F>fu{%uX9xzp8p_&@$HjZOH6f2n__y&nE= z$2$Dmnn-h}tM%}I{9hWI@DKlK_jPIi@qatk;osIonmb*shyUaM(%6K5_?P-;+Uw!} zcC5p{t%)>ux>^ta$N#0V3IFh)c3+qFAOE*w9sX@iq`A}8diX#7FO5z3hkvPmroA5i zZ^t_P+nPvor>phwfBat>oA3|+Y4>$$|M7o2*5TjQM4CHYt%v{P|I*lmfB2XBXWHxG z|8}gyzpaThce+{+|HuEOu?he1pLSoD_8Hm-Zk3w__dtZB3-P)75(TKmIR`P56g@seh)u z9{z8~I{e$3NOPyF_3(fEUmBb65C3WRb!q?ce>>LU-_}H$J6)}Z|KtDC*o1%hm-=Vg z>*4=)ti!*pi8Ob*S`Yun|D~}B|L~u7UzheD|F>g31N^tom~_E2l(QP#Jagj1Ctm!~ zGn8@-{=WWmJBgFCaiX=j=CTRX3;dw~3ZMWApa2S>01BW03ZMWApa2S>01BW03ZMWA zpa2S>01BW03ZMWApa2S>01BW03ZMWApa2S>01BW03ZMWApa2S>01BW03ZMWApa2S> z01BW03ZMWApa2S>01BW03ZMWApa2S>01BW03ZMWApa2S>01BW03ZMWApa2S>01BW0 z3ZMWApa2S>01BW03ZMWApa2S>01BW03ZMWApa2S>01BW03ZMWApa2S>01BW03ZMWA ppa2S>01BW03ZMWApa2S>01BW03ZMWApa2S>01BW03XEET{|7Z-_HzIL literal 0 HcmV?d00001 diff --git a/requirements.txt b/requirements.txt index bf83680..c4d96d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,2 @@ -gdal pillow -numpy httpx diff --git a/tms2geotiff.py b/tms2geotiff.py index f17ff35..e5508d1 100644 --- a/tms2geotiff.py +++ b/tms2geotiff.py @@ -5,13 +5,15 @@ import os import re import math +import time import argparse -import warnings import itertools import concurrent.futures -import numpy from PIL import Image +from PIL import TiffImagePlugin +Image.MAX_IMAGE_PIXELS = None + try: import httpx SESSION = httpx.Client() @@ -31,8 +33,6 @@ EARTH_EQUATORIAL_RADIUS = 6378137.0 -Image.MAX_IMAGE_PIXELS = None - DEFAULT_TMS = 'https://tile.openstreetmap.org/{z}/{x}/{y}.png' @@ -118,7 +118,8 @@ def print_progress(progress, total, done=False): def download_extent( source, lat0, lon0, lat1, lon1, zoom, - progress_callback=print_progress + progress_callback=print_progress, + callback_interval=0.05 ): x0, y0 = deg2num(lat0, lon0, zoom) x1, y1 = deg2num(lat1, lon1, zoom) @@ -130,18 +131,42 @@ def download_extent( range(math.floor(x0), math.ceil(x1)), range(math.floor(y0), math.ceil(y1)))) totalnum = len(corners) - futures = [] + futures = {} + done_num = 0 + progress_callback(done_num, totalnum, False) + last_done_num = 0 + last_callback = time.monotonic() + cancelled = False with concurrent.futures.ThreadPoolExecutor(5) as executor: for x, y in corners: - futures.append(executor.submit(get_tile, - source.format(z=zoom, x=x, y=y))) + future = executor.submit(get_tile, source.format(z=zoom, x=x, y=y)) + futures[future] = (x, y) bbox = (math.floor(x0), math.floor(y0), math.ceil(x1), math.ceil(y1)) bigim = None base_size = [256, 256] - for k, (fut, corner_xy) in enumerate(zip(futures, corners), 1): - progress_callback(k, totalnum, False) - bigim = paste_tile(bigim, base_size, fut.result(), corner_xy, bbox) - progress_callback(k, totalnum, True) + while futures: + done, not_done = concurrent.futures.wait( + futures.keys(), timeout=callback_interval, + return_when=concurrent.futures.FIRST_COMPLETED + ) + for fut in done: + bigim = paste_tile(bigim, base_size, fut.result(), futures[fut], bbox) + del futures[fut] + done_num += 1 + if time.monotonic() > last_callback + callback_interval: + try: + progress_callback(done_num, totalnum, (done_num > last_done_num)) + except TaskCancelled: + for fut in futures.keys(): + fut.cancel() + futures.clear() + cancelled = True + break + last_callback = time.monotonic() + last_done_num = done_num + if cancelled: + raise TaskCancelled() + progress_callback(done_num, totalnum, True) xfrac = x0 - bbox[0] yfrac = y0 - bbox[1] @@ -161,6 +186,61 @@ def download_extent( return retim, matrix +def generate_tiffinfo(matrix): + ifd = TiffImagePlugin.ImageFileDirectory_v2() + # GeoKeyDirectoryTag + gkdt = [ + 1, 1, + 0, # GeoTIFF 1.0 + 0, # NumberOfKeys + ] + # KeyID, TIFFTagLocation, KeyCount, ValueOffset + geokeys = [ + # GTModelTypeGeoKey + (1024, 0, 1, 1), # 2D projected coordinate reference system + # GTRasterTypeGeoKey + (1025, 0, 1, 1), # PixelIsArea + # GTCitationGeoKey + (1026, 34737, 25, 0), + # GeodeticCitationGeoKey + (2049, 34737, 7, 25), + # GeogAngularUnitsGeoKey + (2054, 0, 1, 9102), # degree + # ProjectedCRSGeoKey + (3072, 0, 1, 3857), + # ProjLinearUnitsGeoKey + (3076, 0, 1, 9001), # metre + ] + gkdt[3] = len(geokeys) + ifd.tagtype[34735] = 3 # short + ifd[34735] = tuple(itertools.chain(gkdt, *geokeys)) + # GeoDoubleParamsTag + ifd.tagtype[34736] = 12 # double + # GeoAsciiParamsTag + ifd.tagtype[34737] = 1 # byte + ifd[34737] = b'WGS 84 / Pseudo-Mercator|WGS 84|\x00' + a, b, c, d, e, f = matrix + # ModelPixelScaleTag + ifd.tagtype[33550] = 12 # double + # ModelTiepointTag + ifd.tagtype[33922] = 12 # double + # ModelTransformationTag + ifd.tagtype[34264] = 12 # double + # This matrix tag should not be used + # if the ModelTiepointTag and the ModelPixelScaleTag are already defined + if c == 0 and e == 0: + ifd[33550] = (b, -f, 0.0) + ifd[33922] = (0.0, 0.0, 0.0, a, d, 0.0) + else: + ifd[34264] = ( + b, c, 0.0, a, + e, f, 0.0, d, + 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 1.0 + ) + return ifd + + def save_image(img, filename, matrix, **params): wld_ext = { '.gif': '.gfw', @@ -181,15 +261,23 @@ def save_image(img, filename, matrix, **params): elif ext == '.png': img_params['optimize'] = True elif ext.startswith('.tif'): - img_params['compression'] = 'tiff_lzw' + img_params['compression'] = 'tiff_adobe_deflate' + img_params['tiffinfo'] = generate_tiffinfo(matrix) img.save(filename, **img_params) - with open(wld_name, 'w', encoding='utf-8') as f_wld: - a, b, c, d, e, f = matrix - f_wld.write('\n'.join(map(str, (b, e, c, f, a, d, '')))) + if not ext.startswith('.tif'): + with open(wld_name, 'w', encoding='utf-8') as f_wld: + a, b, c, d, e, f = matrix + f_wld.write('\n'.join(map(str, (b, e, c, f, a, d, '')))) return img -def save_geotiff(img, filename, matrix): +def save_geotiff_gdal(img, filename, matrix): + if 'GDAL_DATA' in os.environ: + del os.environ['GDAL_DATA'] + if 'PROJ_LIB' in os.environ: + del os.environ['PROJ_LIB'] + + import numpy from osgeo import gdal gdal.UseExceptions() @@ -209,18 +297,12 @@ def save_geotiff(img, filename, matrix): return img -def save_image_auto(img, filename, matrix, use_geotiff=False, **params): +def save_image_auto(img, filename, matrix, use_gdal=False, **params): ext = os.path.splitext(filename)[1].lower() - if ext not in ('.tif', '.tiff'): + if ext in ('.tif', '.tiff') and use_gdal: + return save_geotiff_gdal(img, filename, matrix) + else: return save_image(img, filename, matrix, **params) - try: - save_geotiff(img, filename, matrix) - except (ImportError, RuntimeError) as ex: - if use_geotiff: - raise - warnings.warn("Can't use gdal to save GeoTIFF, %s: %s" % ( - type(ex).__name__, ex), RuntimeWarning) - save_image(img, filename, matrix, **params) class TaskCancelled(RuntimeError): @@ -242,11 +324,11 @@ def gui(): def cmd_get_save_file(): result = root_tk.tk.eval("""tk_getSaveFile -filetypes { - {{PNG} {.png}} - {{TIFF} {.tiff}} + {{GeoTIFF} {.tiff}} {{JPG} {.jpg}} + {{PNG} {.png}} {{All Files} *} - } -defaultextension .png""") + } -defaultextension .tiff""") if result: v_output.set(result) @@ -282,6 +364,7 @@ def cmd_get_save_file(): p_progress = ttk.Progressbar(frame, mode='determinate') p_progress.grid(column=0, row=6, columnspan=3, sticky='we', pady=(5, 2)) + started = False stop_download = False def reset(): @@ -290,26 +373,34 @@ def reset(): root_tk.update() def update_progress(progress, total, done): - nonlocal stop_download - if done: + nonlocal started, stop_download + if not started: + if done: + p_progress.configure(maximum=total, value=progress) + else: + p_progress.configure(maximum=total) + started = True + elif done: p_progress.configure(value=progress) - else: - p_progress.configure(maximum=total) root_tk.update() if stop_download: raise TaskCancelled() def cmd_download(): - nonlocal stop_download + nonlocal started, stop_download + started = False stop_download = False b_download.configure(text='Cancel', command=cmd_cancel) root_tk.update() try: - args = [v_url.get().strip()] + url = v_url.get().strip() + args = [url] args.extend(parse_extent(v_extent.get())) args.append(int(v_zoom.get())) filename = v_output.get() - except (TypeError, ValueError) as ex: + if not all(args) or not filename: + raise ValueError("Empty input") + except (TypeError, ValueError, IndexError) as ex: reset() tkinter.messagebox.showerror( title='tms2geotiff', @@ -348,7 +439,8 @@ def cmd_download(): ) def cmd_cancel(): - nonlocal stop_download + nonlocal started, stop_download + started = False stop_download = True reset()