Skip to content
This repository was archived by the owner on Aug 13, 2019. It is now read-only.

Commit 3ee616a

Browse files
committed
Merge remote-tracking branch 'upstream/master' into fix_seek
2 parents fff4e5f + 77cc2ea commit 3ee616a

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

62 files changed

+7361
-1795
lines changed

.travis.yml

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,24 @@
1-
sudo: false
2-
1+
# sudo is enabled because it provides more memory which was needed to run go test -race
2+
sudo: required
3+
dist: trusty
34
language: go
5+
os:
6+
- windows
7+
- linux
8+
- osx
49

510
go:
6-
- 1.9.x
7-
- 1.10.x
11+
- 1.10.x
12+
- 1.11.x
813

914
go_import_path: github.com/prometheus/tsdb
1015

16+
before_install:
17+
- if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then choco install make; fi
18+
19+
install:
20+
- make deps
21+
1122
script:
12-
- go test -timeout 5m ./...
23+
# `staticcheck` target is omitted due to linting errors
24+
- if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then make test; else make; fi

CHANGELOG.md

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
## master / unreleased
2+
3+
- [CHANGE] `prometheus_tsdb_storage_blocks_bytes_total` is now `prometheus_tsdb_storage_blocks_bytes`
4+
5+
## 0.4.0
6+
- [CHANGE] New `WALSegmentSize` option to override the `DefaultOptions.WALSegmentSize`. Added to allow using smaller wal files. For example using tmpfs on a RPI to minimise the SD card wear out from the constant WAL writes. As part of this change the `DefaultOptions.WALSegmentSize` constant was also exposed.
7+
- [CHANGE] Empty blocks are not written during compaction [#374](https://github.com/prometheus/tsdb/pull/374)
8+
- [FEATURE] Size base retention through `Options.MaxBytes`. As part of this change:
9+
- added new metrics - `prometheus_tsdb_storage_blocks_bytes_total`, `prometheus_tsdb_size_retentions_total`, `prometheus_tsdb_time_retentions_total`
10+
- new public interface `SizeReader: Size() int64`
11+
- `OpenBlock` signature changed to take a logger.
12+
- [REMOVED] `PrefixMatcher` is considered unused so was removed.
13+
- [CLEANUP] `Options.WALFlushInterval` is removed as it wasn't used anywhere.
14+
- [FEATURE] Add new `LiveReader` to WAL pacakge. Added to allow live tailing of a WAL segment, used by Prometheus Remote Write after refactor. The main difference between the new reader and the existing `Reader` is that for `LiveReader` a call to `Next()` that returns false does not mean that there will never be more data to read.
15+
16+
## 0.3.1
17+
- [BUGFIX] Fixed most windows test and some actual bugs for unclosed file readers.
18+
19+
## 0.3.0
20+
- [CHANGE] `LastCheckpoint()` used to return just the segment name and now it returns the full relative path.
21+
- [CHANGE] `NewSegmentsRangeReader()` can now read over miltiple wal ranges by using the new `SegmentRange{}` struct.
22+
- [CHANGE] `CorruptionErr{}` now also exposes the Segment `Dir` which is added when displaying any errors.
23+
- [CHANGE] `Head.Init()` is changed to `Head.Init(minValidTime int64)`
24+
- [CHANGE] `SymbolTable()` renamed to `SymbolTableSize()` to make the name consistent with the `Block{ symbolTableSize uint64 }` field.
25+
- [CHANGE] `wal.Reader{}` now exposes `Segment()` for the current segment being read and `Offset()` for the current offset.
26+
-[FEATURE] tsdbutil analyze subcomand to find churn, high cardinality, etc.

MAINTAINERS.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Maintainers of this repository:
2+
3+
* Krasi Georgiev <[email protected]> @krasi-georgiev
4+
* Goutham Veeramachaneni <[email protected]> @gouthamve

Makefile

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# Copyright 2018 The Prometheus Authors
2+
# Licensed under the Apache License, Version 2.0 (the "License");
3+
# you may not use this file except in compliance with the License.
4+
# You may obtain a copy of the License at
5+
#
6+
# http://www.apache.org/licenses/LICENSE-2.0
7+
#
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an "AS IS" BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
14+
TSDB_PROJECT_DIR = "."
15+
TSDB_CLI_DIR="$(TSDB_PROJECT_DIR)/cmd/tsdb"
16+
TSDB_BIN = "$(TSDB_CLI_DIR)/tsdb"
17+
TSDB_BENCHMARK_NUM_METRICS ?= 1000
18+
TSDB_BENCHMARK_DATASET ?= "$(TSDB_PROJECT_DIR)/testdata/20kseries.json"
19+
TSDB_BENCHMARK_OUTPUT_DIR ?= "$(TSDB_CLI_DIR)/benchout"
20+
21+
include Makefile.common
22+
23+
.PHONY: deps
24+
deps:
25+
@echo ">> getting dependencies"
26+
GO111MODULE=$(GO111MODULE) $(GO) get $(GOOPTS) -t ./...
27+
28+
build:
29+
GO111MODULE=$(GO111MODULE) $(GO) build -o $(TSDB_BIN) $(TSDB_CLI_DIR)
30+
31+
bench: build
32+
@echo ">> running benchmark, writing result to $(TSDB_BENCHMARK_OUTPUT_DIR)"
33+
@$(TSDB_BIN) bench write --metrics=$(TSDB_BENCHMARK_NUM_METRICS) --out=$(TSDB_BENCHMARK_OUTPUT_DIR) $(TSDB_BENCHMARK_DATASET)
34+
@$(GO) tool pprof -svg $(TSDB_BIN) $(TSDB_BENCHMARK_OUTPUT_DIR)/cpu.prof > $(TSDB_BENCHMARK_OUTPUT_DIR)/cpuprof.svg
35+
@$(GO) tool pprof --inuse_space -svg $(TSDB_BIN) $(TSDB_BENCHMARK_OUTPUT_DIR)/mem.prof > $(TSDB_BENCHMARK_OUTPUT_DIR)/memprof.inuse.svg
36+
@$(GO) tool pprof --alloc_space -svg $(TSDB_BIN) $(TSDB_BENCHMARK_OUTPUT_DIR)/mem.prof > $(TSDB_BENCHMARK_OUTPUT_DIR)/memprof.alloc.svg
37+
@$(GO) tool pprof -svg $(TSDB_BIN) $(TSDB_BENCHMARK_OUTPUT_DIR)/block.prof > $(TSDB_BENCHMARK_OUTPUT_DIR)/blockprof.svg
38+
@$(GO) tool pprof -svg $(TSDB_BIN) $(TSDB_BENCHMARK_OUTPUT_DIR)/mutex.prof > $(TSDB_BENCHMARK_OUTPUT_DIR)/mutexprof.svg

Makefile.common

Lines changed: 223 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,223 @@
1+
# Copyright 2018 The Prometheus Authors
2+
# Licensed under the Apache License, Version 2.0 (the "License");
3+
# you may not use this file except in compliance with the License.
4+
# You may obtain a copy of the License at
5+
#
6+
# http://www.apache.org/licenses/LICENSE-2.0
7+
#
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an "AS IS" BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
14+
15+
# A common Makefile that includes rules to be reused in different prometheus projects.
16+
# !!! Open PRs only against the prometheus/prometheus/Makefile.common repository!
17+
18+
# Example usage :
19+
# Create the main Makefile in the root project directory.
20+
# include Makefile.common
21+
# customTarget:
22+
# @echo ">> Running customTarget"
23+
#
24+
25+
# Ensure GOBIN is not set during build so that promu is installed to the correct path
26+
unexport GOBIN
27+
28+
GO ?= go
29+
GOFMT ?= $(GO)fmt
30+
FIRST_GOPATH := $(firstword $(subst :, ,$(shell $(GO) env GOPATH)))
31+
GOOPTS ?=
32+
33+
GO_VERSION ?= $(shell $(GO) version)
34+
GO_VERSION_NUMBER ?= $(word 3, $(GO_VERSION))
35+
PRE_GO_111 ?= $(shell echo $(GO_VERSION_NUMBER) | grep -E 'go1\.(10|[0-9])\.')
36+
37+
unexport GOVENDOR
38+
ifeq (, $(PRE_GO_111))
39+
ifneq (,$(wildcard go.mod))
40+
# Enforce Go modules support just in case the directory is inside GOPATH (and for Travis CI).
41+
GO111MODULE := on
42+
43+
ifneq (,$(wildcard vendor))
44+
# Always use the local vendor/ directory to satisfy the dependencies.
45+
GOOPTS := $(GOOPTS) -mod=vendor
46+
endif
47+
endif
48+
else
49+
ifneq (,$(wildcard go.mod))
50+
ifneq (,$(wildcard vendor))
51+
$(warning This repository requires Go >= 1.11 because of Go modules)
52+
$(warning Some recipes may not work as expected as the current Go runtime is '$(GO_VERSION_NUMBER)')
53+
endif
54+
else
55+
# This repository isn't using Go modules (yet).
56+
GOVENDOR := $(FIRST_GOPATH)/bin/govendor
57+
endif
58+
59+
unexport GO111MODULE
60+
endif
61+
PROMU := $(FIRST_GOPATH)/bin/promu
62+
STATICCHECK := $(FIRST_GOPATH)/bin/staticcheck
63+
pkgs = ./...
64+
65+
GO_VERSION ?= $(shell $(GO) version)
66+
GO_BUILD_PLATFORM ?= $(subst /,-,$(lastword $(GO_VERSION)))
67+
68+
PROMU_VERSION ?= 0.2.0
69+
PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz
70+
71+
PREFIX ?= $(shell pwd)
72+
BIN_DIR ?= $(shell pwd)
73+
DOCKER_IMAGE_TAG ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD))
74+
DOCKER_REPO ?= prom
75+
76+
.PHONY: all
77+
all: precheck style staticcheck unused build test
78+
79+
# This rule is used to forward a target like "build" to "common-build". This
80+
# allows a new "build" target to be defined in a Makefile which includes this
81+
# one and override "common-build" without override warnings.
82+
%: common-% ;
83+
84+
.PHONY: common-style
85+
common-style:
86+
@echo ">> checking code style"
87+
@fmtRes=$$($(GOFMT) -d $$(find . -path ./vendor -prune -o -name '*.go' -print)); \
88+
if [ -n "$${fmtRes}" ]; then \
89+
echo "gofmt checking failed!"; echo "$${fmtRes}"; echo; \
90+
echo "Please ensure you are using $$($(GO) version) for formatting code."; \
91+
exit 1; \
92+
fi
93+
94+
.PHONY: common-check_license
95+
common-check_license:
96+
@echo ">> checking license header"
97+
@licRes=$$(for file in $$(find . -type f -iname '*.go' ! -path './vendor/*') ; do \
98+
awk 'NR<=3' $$file | grep -Eq "(Copyright|generated|GENERATED)" || echo $$file; \
99+
done); \
100+
if [ -n "$${licRes}" ]; then \
101+
echo "license header checking failed:"; echo "$${licRes}"; \
102+
exit 1; \
103+
fi
104+
105+
.PHONY: common-test-short
106+
common-test-short:
107+
@echo ">> running short tests"
108+
GO111MODULE=$(GO111MODULE) $(GO) test -short $(GOOPTS) $(pkgs)
109+
110+
.PHONY: common-test
111+
common-test:
112+
@echo ">> running all tests"
113+
GO111MODULE=$(GO111MODULE) $(GO) test -race $(GOOPTS) $(pkgs)
114+
115+
.PHONY: common-format
116+
common-format:
117+
@echo ">> formatting code"
118+
GO111MODULE=$(GO111MODULE) $(GO) fmt $(GOOPTS) $(pkgs)
119+
120+
.PHONY: common-vet
121+
common-vet:
122+
@echo ">> vetting code"
123+
GO111MODULE=$(GO111MODULE) $(GO) vet $(GOOPTS) $(pkgs)
124+
125+
.PHONY: common-staticcheck
126+
common-staticcheck: $(STATICCHECK)
127+
@echo ">> running staticcheck"
128+
ifdef GO111MODULE
129+
GO111MODULE=$(GO111MODULE) $(STATICCHECK) -ignore "$(STATICCHECK_IGNORE)" -checks "SA*" $(pkgs)
130+
else
131+
$(STATICCHECK) -ignore "$(STATICCHECK_IGNORE)" $(pkgs)
132+
endif
133+
134+
.PHONY: common-unused
135+
common-unused: $(GOVENDOR)
136+
ifdef GOVENDOR
137+
@echo ">> running check for unused packages"
138+
@$(GOVENDOR) list +unused | grep . && exit 1 || echo 'No unused packages'
139+
else
140+
ifdef GO111MODULE
141+
@echo ">> running check for unused/missing packages in go.mod"
142+
GO111MODULE=$(GO111MODULE) $(GO) mod tidy
143+
@git diff --exit-code -- go.sum go.mod
144+
ifneq (,$(wildcard vendor))
145+
@echo ">> running check for unused packages in vendor/"
146+
GO111MODULE=$(GO111MODULE) $(GO) mod vendor
147+
@git diff --exit-code -- go.sum go.mod vendor/
148+
endif
149+
endif
150+
endif
151+
152+
.PHONY: common-build
153+
common-build: promu
154+
@echo ">> building binaries"
155+
GO111MODULE=$(GO111MODULE) $(PROMU) build --prefix $(PREFIX)
156+
157+
.PHONY: common-tarball
158+
common-tarball: promu
159+
@echo ">> building release tarball"
160+
$(PROMU) tarball --prefix $(PREFIX) $(BIN_DIR)
161+
162+
.PHONY: common-docker
163+
common-docker:
164+
docker build -t "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)" .
165+
166+
.PHONY: common-docker-publish
167+
common-docker-publish:
168+
docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)"
169+
170+
.PHONY: common-docker-tag-latest
171+
common-docker-tag-latest:
172+
docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):latest"
173+
174+
.PHONY: promu
175+
promu: $(PROMU)
176+
177+
$(PROMU):
178+
curl -s -L $(PROMU_URL) | tar -xvz -C /tmp
179+
mkdir -v -p $(FIRST_GOPATH)/bin
180+
cp -v /tmp/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM)/promu $(PROMU)
181+
182+
.PHONY: proto
183+
proto:
184+
@echo ">> generating code from proto files"
185+
@./scripts/genproto.sh
186+
187+
.PHONY: $(STATICCHECK)
188+
$(STATICCHECK):
189+
ifdef GO111MODULE
190+
# Get staticcheck from a temporary directory to avoid modifying the local go.{mod,sum}.
191+
# See https://github.com/golang/go/issues/27643.
192+
# For now, we are using the next branch of staticcheck because master isn't compatible yet with Go modules.
193+
tmpModule=$$(mktemp -d 2>&1) && \
194+
mkdir -p $${tmpModule}/staticcheck && \
195+
cd "$${tmpModule}"/staticcheck && \
196+
GO111MODULE=on $(GO) mod init example.com/staticcheck && \
197+
GO111MODULE=on GOOS= GOARCH= $(GO) get -u honnef.co/go/tools/cmd/staticcheck@next && \
198+
rm -rf $${tmpModule};
199+
else
200+
GOOS= GOARCH= GO111MODULE=off $(GO) get -u honnef.co/go/tools/cmd/staticcheck
201+
endif
202+
203+
ifdef GOVENDOR
204+
.PHONY: $(GOVENDOR)
205+
$(GOVENDOR):
206+
GOOS= GOARCH= $(GO) get -u github.com/kardianos/govendor
207+
endif
208+
209+
.PHONY: precheck
210+
precheck::
211+
212+
define PRECHECK_COMMAND_template =
213+
precheck:: $(1)_precheck
214+
215+
216+
PRECHECK_COMMAND_$(1) ?= $(1) $$(strip $$(PRECHECK_OPTIONS_$(1)))
217+
.PHONY: $(1)_precheck
218+
$(1)_precheck:
219+
@if ! $$(PRECHECK_COMMAND_$(1)) 1>/dev/null 2>&1; then \
220+
echo "Execution of '$$(PRECHECK_COMMAND_$(1))' command failed. Is $(1) installed?"; \
221+
exit 1; \
222+
fi
223+
endef

README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ This repository contains the Prometheus storage layer that is used in its 2.x re
77

88
A writeup of its design can be found [here](https://fabxc.org/blog/2017-04-10-writing-a-tsdb/).
99

10+
Based on the Gorilla TSDB [white papers](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf).
11+
1012
Video: [Storing 16 Bytes at Scale](https://youtu.be/b_pEevMAC3I) from [PromCon 2017](https://promcon.io/2017-munich/).
1113

1214
See also the [format documentation](docs/format/README.md).

0 commit comments

Comments
 (0)