修复sqlite数据库初始化错误

Author: atlasdatatech

go.mod

@@ -5,11 +5,12 @@ go 1.15
 require (
 	github.com/antonfisher/nested-logrus-formatter v1.3.0
 	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/mattn/go-sqlite3 v1.14.19
 	github.com/paulmach/orb v0.1.6
-	github.com/shiena/ansicolor v0.0.0-20200830101100-9405ca8e49f3
+	github.com/shiena/ansicolor v0.0.0-20230509054315-a9deabde6e02
 	github.com/sirupsen/logrus v1.6.0
 	github.com/spf13/viper v1.7.1
-	github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf
+	github.com/teris-io/shortid v0.0.0-20220617161101-71ec9f2aa569
 	golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd // indirect
 	gopkg.in/cheggaaa/pb.v1 v1.0.28
 )

go.sum

@@ -124,6 +124,8 @@ github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI=
+github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
@@ -159,8 +161,8 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
-github.com/shiena/ansicolor v0.0.0-20200830101100-9405ca8e49f3 h1:AutMBemBfeplnEpI2sXdLuvQKKyHixlKKSBFCcPLaE4=
-github.com/shiena/ansicolor v0.0.0-20200830101100-9405ca8e49f3/go.mod h1:nkxAfR/5quYxwPZhyDxgasBMnRtBZd0FCEpawpjMUFg=
+github.com/shiena/ansicolor v0.0.0-20230509054315-a9deabde6e02 h1:v9ezJDHA1XGxViAUSIoO/Id7Fl63u6d0YmsAm+/p2hs=
+github.com/shiena/ansicolor v0.0.0-20230509054315-a9deabde6e02/go.mod h1:RF16/A3L0xSa0oSERcnhd8Pu3IXSDZSK2gmGIMsttFE=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
@@ -187,8 +189,8 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
-github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf h1:Z2X3Os7oRzpdJ75iPqWZc0HeJWFYNCvKsfpQwFpRNTA=
-github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf/go.mod h1:M8agBzgqHIhgj7wEn9/0hJUZcrvt9VY+Ln+S1I5Mha0=
+github.com/teris-io/shortid v0.0.0-20220617161101-71ec9f2aa569 h1:xzABM9let0HLLqFypcxvLmlvEciCHL7+Lv+4vwZqecI=
+github.com/teris-io/shortid v0.0.0-20220617161101-71ec9f2aa569/go.mod h1:2Ly+NIftZN4de9zRmENdYbvPQeaVIYKWpLFStLFEBgI=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=

main.go

@@ -10,6 +10,7 @@ import (
 	log "github.com/sirupsen/logrus"
 
 	nested "github.com/antonfisher/nested-logrus-formatter"
+	_ "github.com/mattn/go-sqlite3"
 	"github.com/spf13/viper"
 )
 

map.go

@@ -8,7 +8,7 @@ import (
 	"github.com/paulmach/orb/maptile"
 )
 
-//TileMap 瓦片地图类型
+// TileMap 瓦片地图类型
 type TileMap struct {
 	ID          int
 	Name        string
@@ -23,14 +23,14 @@ type TileMap struct {
 	//such as porxy...
 }
 
-//CreateTileMap 添加地图
+// CreateTileMap 添加地图
 func CreateTileMap(url string) {
 	// tileMap := TileMap{}
 	//tileMap.Save()
 	//成功默认保存到数据库
 }
 
-//GetTileMapList 获取初始化默认地图列表
+// GetTileMapList 获取初始化默认地图列表
 func GetTileMapList() map[int]TileMap {
 	tml := make(map[int]TileMap)
 	var list = []string{"http://mt0.google.com/vt/lyrs=y&x={x}&y={y}&z={z}",
@@ -49,7 +49,7 @@ func GetTileMapList() map[int]TileMap {
 	return tml
 }
 
-//TileURL 获取瓦片URL
+// TileURL 获取瓦片URL
 func (m TileMap) getTileURL(t maptile.Tile) string {
 	url := strings.Replace(m.URL, "{x}", strconv.Itoa(int(t.X)), -1)
 	url = strings.Replace(url, "{y}", strconv.Itoa(int(t.Y)), -1)

task.go

@@ -5,7 +5,8 @@ import (
 	"compress/gzip"
 	"database/sql"
 	"fmt"
-	"io/ioutil"
+	"io"
+	"math"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -23,10 +24,10 @@ import (
 	pb "gopkg.in/cheggaaa/pb.v1"
 )
 
-//MBTileVersion mbtiles版本号
+// MBTileVersion mbtiles版本号
 const MBTileVersion = "1.2"
 
-//Task 下载任务
+// Task 下载任务
 type Task struct {
 	ID                 string
 	Name               string
@@ -52,7 +53,7 @@ type Task struct {
 	outformat          string
 }
 
-//NewTask 创建下载任务
+// NewTask 创建下载任务
 func NewTask(layers []Layer, m TileMap) *Task {
 	if len(layers) == 0 {
 		return nil
@@ -92,7 +93,7 @@ func NewTask(layers []Layer, m TileMap) *Task {
 	return &task
 }
 
-//Bound 范围
+// Bound 范围
 func (task *Task) Bound() orb.Bound {
 	bound := orb.Bound{Min: orb.Point{1, 1}, Max: orb.Point{-1, -1}}
 	for _, layer := range task.Layers {
@@ -103,7 +104,7 @@ func (task *Task) Bound() orb.Bound {
 	return bound
 }
 
-//Center 中心点
+// Center 中心点
 func (task *Task) Center() orb.Point {
 	layer := task.Layers[len(task.Layers)-1]
 	bound := orb.Bound{Min: orb.Point{1, 1}, Max: orb.Point{-1, -1}}
@@ -113,7 +114,7 @@ func (task *Task) Center() orb.Point {
 	return bound.Center()
 }
 
-//MetaItems 输出
+// MetaItems 输出
 func (task *Task) MetaItems() map[string]string {
 	b := task.Bound()
 	c := task.Center()
@@ -136,7 +137,7 @@ func (task *Task) MetaItems() map[string]string {
 	return data
 }
 
-//SetupMBTileTables 初始化配置MBTile库
+// SetupMBTileTables 初始化配置MBTile库
 func (task *Task) SetupMBTileTables() error {
 
 	if task.File == "" {
@@ -205,7 +206,7 @@ func (task *Task) playFun() {
 	task.play <- struct{}{}
 }
 
-//SavePipe 保存瓦片管道
+// SavePipe 保存瓦片管道
 func (task *Task) savePipe() {
 	for tile := range task.savingpipe {
 		err := saveToMBTile(tile, task.db)
@@ -219,7 +220,7 @@ func (task *Task) savePipe() {
 	}
 }
 
-//SaveTile 保存瓦片
+// SaveTile 保存瓦片
 func (task *Task) saveTile(tile Tile) error {
 	// defer task.wg.Done()
 	err := saveToFiles(tile, task)
@@ -229,7 +230,7 @@ func (task *Task) saveTile(tile Tile) error {
 	return nil
 }
 
-//tileFetcher 瓦片加载器
+// tileFetcher 瓦片加载器
 func (task *Task) tileFetcher(mt maptile.Tile, url string) {
 	start := time.Now()
 	defer task.tileWG.Done() //结束该瓦片请求
@@ -240,21 +241,44 @@ func (task *Task) tileFetcher(mt maptile.Tile, url string) {
 	prep := func(t maptile.Tile, url string) string {
 		url = strings.Replace(url, "{x}", strconv.Itoa(int(t.X)), -1)
 		url = strings.Replace(url, "{y}", strconv.Itoa(int(t.Y)), -1)
+		maxY := int(math.Pow(2, float64(t.Z))) - 1
+		url = strings.Replace(url, "{-y}", strconv.Itoa(maxY-int(t.Y)), -1)
 		url = strings.Replace(url, "{z}", strconv.Itoa(int(t.Z)), -1)
 		return url
 	}
 	tile := prep(mt, url)
-	resp, err := http.Get(tile)
+	client := &http.Client{
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			// 自定义重定向的行为
+			return http.ErrUseLastResponse // 使用最后一个响应
+		},
+	}
+	req, err := http.NewRequest("GET", tile, nil)
+	if err != nil {
+		fmt.Println("创建请求失败:", err)
+		return
+	}
+
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
+	// req.Header.Set("Origin", "https://map.tianditu.gov.cn")
+	req.Header.Set("Referer", "https://jiangsu.tianditu.gov.cn")
+	resp, err := client.Do(req)
 	if err != nil {
-		log.Errorf("fetch :%s error, details: %s ~", tile, err)
+		fmt.Println("发送请求失败:", err)
 		return
 	}
 	defer resp.Body.Close()
+	// resp, err := http.Get(tile)
+	// if err != nil {
+	// 	log.Errorf("fetch :%s error, details: %s ~", tile, err)
+	// 	return
+	// }
+	// defer resp.Body.Close()
 	if resp.StatusCode != 200 {
 		log.Errorf("fetch %v tile error, status code: %d ~", tile, resp.StatusCode)
 		return
 	}
-	body, err := ioutil.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		log.Errorf("read %v tile error ~ %s", mt, err)
 		return
@@ -294,7 +318,7 @@ func (task *Task) tileFetcher(mt maptile.Tile, url string) {
 	log.Infof("tile(z:%d, x:%d, y:%d), %dms , %.2f kb, %s ...\n", mt.Z, mt.X, mt.Y, cost, float32(len(body))/1024.0, tile)
 }
 
-//DownloadZoom 下载指定层级
+// DownloadZoom 下载指定层级
 func (task *Task) downloadLayer(layer Layer) {
 	bar := pb.New64(layer.Count).Prefix(fmt.Sprintf("Zoom %d : ", layer.Zoom)).Postfix("\n")
 	// bar.SetRefreshRate(time.Second)
@@ -334,7 +358,7 @@ func (task *Task) downloadLayer(layer Layer) {
 	bar.FinishPrint(fmt.Sprintf("Task %s Zoom %d finished ~", task.ID, layer.Zoom))
 }
 
-//Download 开启下载任务
+// Download 开启下载任务
 func (task *Task) Download() {
 	//g orb.Geometry, minz int, maxz int
 	task.Bar = pb.New64(task.Total).Prefix("Task : ").Postfix("\n")

utils.go

@@ -4,7 +4,6 @@ import (
 	"database/sql"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sync"
@@ -30,7 +29,7 @@ func saveToFiles(tile Tile, task *Task) error {
 	dir := filepath.Join(rootdir, fmt.Sprintf(`%d`, tile.T.Z), fmt.Sprintf(`%d`, tile.T.X))
 	os.MkdirAll(dir, os.ModePerm)
 	fileName := filepath.Join(dir, fmt.Sprintf(`%d.%s`, tile.T.Y, task.TileMap.Format))
-	err := ioutil.WriteFile(fileName, tile.C, os.ModePerm)
+	err := os.WriteFile(fileName, tile.C, os.ModePerm)
 	if err != nil {
 		return err
 	}
@@ -69,7 +68,7 @@ func optimizeDatabase(db *sql.DB) error {
 }
 
 func loadFeature(path string) *geojson.Feature {
-	data, err := ioutil.ReadFile(path)
+	data, err := os.ReadFile(path)
 	if err != nil {
 		log.Fatalf("unable to read file: %v", err)
 	}
@@ -96,7 +95,7 @@ func loadFeature(path string) *geojson.Feature {
 }
 
 func loadFeatureCollection(path string) *geojson.FeatureCollection {
-	data, err := ioutil.ReadFile(path)
+	data, err := os.ReadFile(path)
 	if err != nil {
 		log.Fatalf("unable to read file: %v", err)
 	}
@@ -119,7 +118,7 @@ func loadFeatureCollection(path string) *geojson.FeatureCollection {
 }
 
 func loadCollection(path string) orb.Collection {
-	data, err := ioutil.ReadFile(path)
+	data, err := os.ReadFile(path)
 	if err != nil {
 		log.Fatalf("unable to read file: %v", err)
 	}
@@ -155,7 +154,7 @@ func output(name string, r *geojson.FeatureCollection) {
 		log.Fatalf("error marshalling json: %v", err)
 	}
 
-	err = ioutil.WriteFile("failure_"+name+".geojson", data, 0644)
+	err = os.WriteFile("failure_"+name+".geojson", data, 0644)
 	if err != nil {
 		log.Fatalf("write file failure: %v", err)
 	}
@@ -169,7 +168,7 @@ func output2(name string, r *geojson.FeatureCollection, wg *sync.WaitGroup) {
 		log.Fatalf("error marshalling json: %v", err)
 	}
 
-	err = ioutil.WriteFile(name+".geojson", data, 0644)
+	err = os.WriteFile(name+".geojson", data, 0644)
 	if err != nil {
 		log.Fatalf("write file failure: %v", err)
 	}