diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c8c656e02..4d38b6f45 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -1,5 +1,5 @@ name: "Build" -on: [push, pull_request] +on: [pull_request] jobs: build: diff --git a/.github/workflows/build_images.yaml b/.github/workflows/build_images.yaml new file mode 100644 index 000000000..bf4d666da --- /dev/null +++ b/.github/workflows/build_images.yaml @@ -0,0 +1,16 @@ +name: "Build-image" +on: [pull_request] + +jobs: + buildx: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - uses: docker/setup-buildx-action@v1 + id: buildx + with: + install: true + - name: Build + run: | + make image diff --git a/.github/workflows/build_workflow.yaml b/.github/workflows/build_workflow.yaml index 1fed0305a..8660928bc 100644 --- a/.github/workflows/build_workflow.yaml +++ b/.github/workflows/build_workflow.yaml @@ -1,5 +1,5 @@ name: "Build-workflow" -on: [push, pull_request] +on: [pull_request] jobs: build: diff --git a/.github/workflows/precheck.yaml b/.github/workflows/precheck.yaml index ed8506b3a..0a4d8f3f4 100644 --- a/.github/workflows/precheck.yaml +++ b/.github/workflows/precheck.yaml @@ -1,5 +1,5 @@ name: "Pre-Check" -on: [push, pull_request] +on: [pull_request] jobs: pre-check: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5c576dd48..cf70bd6d0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,5 +1,5 @@ name: "Test" -on: [push, pull_request] +on: [pull_request] jobs: test: diff --git a/Makefile b/Makefile index 456ec44c0..4e983bc9d 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ LDFLAGS += -X "github.com/pingcap/tipocket/pkg/test-infra/fixture.BuildHash=$(sh GOBUILD=$(GO) build -ldflags '$(LDFLAGS)' -DOCKER_REGISTRY_PREFIX := $(if $(DOCKER_REGISTRY),$(DOCKER_REGISTRY)/,) +DOCKER_REGISTRY_TIPOCKET := $(if $(DOCKER_REGISTRY),$(DOCKER_REGISTRY)/,)tipocket default: tidy fmt lint build @@ -168,8 +168,9 @@ endif groupimports: install-goimports goimports -w -l -local github.com/pingcap/tipocket $$($(PACKAGE_DIRECTORIES)) +i ?= true init: tipocket - bin/tipocket init -c $(c) + bin/tipocket init -c=$(c) -i=$i install-goimports: ifeq (, $(shell which goimports)) @@ -250,9 +251,11 @@ test: find testcase -mindepth 1 -maxdepth 1 -type d | xargs -I% sh -c 'cd %; make test'; image: - DOCKER_BUILDKIT=1 docker build -t ${DOCKER_REGISTRY_PREFIX}pingcap/tipocket:latest . + DOCKER_BUILDKIT=1 docker build -t ${DOCKER_REGISTRY_TIPOCKET}/tipocket:latest . + find testcase -mindepth 1 -maxdepth 1 -type d | xargs -I% sh -c 'if [ -f %/Dockerfile ]; then DOCKER_BUILDKIT=1 docker build -t ${DOCKER_REGISTRY_TIPOCKET}/`basename %`:latest -f %/Dockerfile .; fi'; docker-push: - docker push ${DOCKER_REGISTRY_PREFIX}pingcap/tipocket:latest + docker push ${DOCKER_REGISTRY_TIPOCKET}/tipocket:latest + find testcase -mindepth 1 -maxdepth 1 -type d | xargs -I% sh -c 'if [ -f %/Dockerfile ]; then DOCKER_BUILDKIT=1 docker push ${DOCKER_REGISTRY_TIPOCKET}/`basename %`:latest; fi'; .PHONY: all clean pocket compare test fmt diff --git a/cmd/tipocket/init.go b/cmd/tipocket/init.go index a82d84114..f482ba99b 100644 --- a/cmd/tipocket/init.go +++ b/cmd/tipocket/init.go @@ -17,7 +17,8 @@ import ( ) var ( - caseNameFlag string + caseNameFlag string + individualBuildFlag bool ) func newInitCmd() *cobra.Command { @@ -35,7 +36,7 @@ func newInitCmd() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { scaffolder := scaffolds.NewScaffold() universe := model.NewUniverse() - err := scaffolder.Execute(universe, &testcase.Makefile{ + fileBuilders := []file.Builder{&testcase.Makefile{ TemplateMixin: file.TemplateMixin{Path: filepath.Join("testcase", caseNameFlag, "Makefile")}, CaseName: caseNameFlag, }, &testcase.Client{ @@ -51,16 +52,24 @@ func newInitCmd() *cobra.Command { TemplateMixin: file.TemplateMixin{Path: filepath.Join("testcase", caseNameFlag, "revive.toml")}, CaseName: caseNameFlag, }, &template.MakefileUpdater{ - InserterMixin: file.InserterMixin{Path: "Makefile"}, - CaseName: caseNameFlag, + InserterMixin: file.InserterMixin{Path: "Makefile"}, + CaseName: caseNameFlag, + IndividualBuild: individualBuildFlag, }, &template.CaseJsonnetUpdater{ InserterMixin: file.InserterMixin{Path: filepath.Join("run", "lib", "case.libsonnet")}, CaseName: caseNameFlag, }, &workflow.CaseJsonnetTemplate{ - TemplateMixin: file.TemplateMixin{Path: filepath.Join("run", "workflow", fmt.Sprintf("%s.jsonnet", caseNameFlag))}, - CaseName: caseNameFlag, - }) - if err != nil { + TemplateMixin: file.TemplateMixin{Path: filepath.Join("run", "workflow", fmt.Sprintf("%s.jsonnet", caseNameFlag))}, + CaseName: caseNameFlag, + IndividualBuild: individualBuildFlag, + }} + if individualBuildFlag { + fileBuilders = append(fileBuilders, &testcase.Dockerfile{ + TemplateMixin: file.TemplateMixin{Path: filepath.Join("testcase", caseNameFlag, "Dockerfile")}, + CaseName: caseNameFlag, + }) + } + if err := scaffolder.Execute(universe, fileBuilders...); err != nil { return err } fmt.Printf("create a new case `%[1]s`: testcase/%[1]s\n", caseNameFlag) @@ -68,6 +77,7 @@ func newInitCmd() *cobra.Command { }, } cmd.Flags().StringVarP(&caseNameFlag, "case-name", "c", "", "test case name") + cmd.Flags().BoolVarP(&individualBuildFlag, "individual-build", "i", false, "individual build from root image") cmd.MarkFlagRequired("case-name") return cmd } diff --git a/pkg/scaffolds/template/makefile.go b/pkg/scaffolds/template/makefile.go index 44468abdf..6b7065c76 100644 --- a/pkg/scaffolds/template/makefile.go +++ b/pkg/scaffolds/template/makefile.go @@ -15,13 +15,18 @@ const ( cd testcase/%[1]s; make build; \ cp bin/* ../../bin/ +` + makefileCmdIndividualBuildInsertionTemplate = `%[1]s: + cd testcase/%[1]s; make build; + ` ) // MakefileUpdater inserts a build rule for the new test case type MakefileUpdater struct { file.InserterMixin - CaseName string + CaseName string + IndividualBuild bool } // GetIfExistsAction ... @@ -31,6 +36,12 @@ func (m *MakefileUpdater) GetIfExistsAction() file.IfExistsAction { // GetCodeFragments ... func (m *MakefileUpdater) GetCodeFragments() map[file.Marker]file.CodeFragment { + if m.IndividualBuild { + return map[file.Marker]file.CodeFragment{ + makefileBuildMarker: {fmt.Sprintf(makefileBuildInsertionTemplate, m.CaseName)}, + makefileCmdMarker: {fmt.Sprintf(makefileCmdIndividualBuildInsertionTemplate, m.CaseName)}, + } + } return map[file.Marker]file.CodeFragment{ makefileBuildMarker: {fmt.Sprintf(makefileBuildInsertionTemplate, m.CaseName)}, makefileCmdMarker: {fmt.Sprintf(makefileCmdInsertionTemplate, m.CaseName)}, diff --git a/pkg/scaffolds/template/testcase/dockerfile.go b/pkg/scaffolds/template/testcase/dockerfile.go new file mode 100644 index 000000000..a3e8f68b2 --- /dev/null +++ b/pkg/scaffolds/template/testcase/dockerfile.go @@ -0,0 +1,66 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package testcase + +import "github.com/pingcap/tipocket/pkg/scaffolds/file" + +// Dockerfile uses for Dockerfile template +type Dockerfile struct { + file.TemplateMixin + CaseName string +} + +// GetIfExistsAction ... +func (g *Dockerfile) GetIfExistsAction() file.IfExistsAction { + return file.IfExistsActionError +} + +// Validate ... +func (g *Dockerfile) Validate() error { + return g.TemplateMixin.Validate() +} + +// SetTemplateDefaults ... +func (g *Dockerfile) SetTemplateDefaults() error { + g.TemplateBody = dockerfileTemplate + return nil +} + +const dockerfileTemplate = `# syntax = docker/dockerfile:1.0-experimental +FROM golang:alpine3.10 AS build_base + +RUN apk add --no-cache gcc libc-dev make bash git + +ENV GO111MODULE=on +WORKDIR /src +COPY . . +COPY .git .git + +RUN rm -rf /go/src/ +RUN --mount=type=cache,id=tipocket_go_pkg,target=/go/pkg \ + --mount=type=cache,id=tipocket_go_cache,target=/root/.cache/go-build \ + --mount=type=tmpfs,id=tipocket_go_src,target=/go/src/ cd testcase/{{ .CaseName }} && make build + +FROM alpine:3.8 + +RUN apk update && apk upgrade && \ + apk add --no-cache bash curl wget + +RUN mkdir -p /config && mkdir -p /resources +COPY --from=0 /src/testcase/{{ .CaseName }}/bin/* /bin/ +COPY --from=0 /src/config /config +COPY --from=0 /src/resources /resources + +EXPOSE 8080 +` diff --git a/pkg/scaffolds/template/workflow/case.jsonnet.go b/pkg/scaffolds/template/workflow/case.jsonnet.go index 5e4eb1b42..bc77c67bb 100644 --- a/pkg/scaffolds/template/workflow/case.jsonnet.go +++ b/pkg/scaffolds/template/workflow/case.jsonnet.go @@ -18,7 +18,8 @@ import "github.com/pingcap/tipocket/pkg/scaffolds/file" // CaseJsonnetTemplate uses for client.go type CaseJsonnetTemplate struct { file.TemplateMixin - CaseName string + CaseName string + IndividualBuild bool } // GetIfExistsAction ... @@ -33,14 +34,19 @@ func (c *CaseJsonnetTemplate) Validate() error { // SetTemplateDefaults ... func (c *CaseJsonnetTemplate) SetTemplateDefaults() error { - c.TemplateBody = clientTemplate + if c.IndividualBuild { + c.TemplateBody = clientIndividualBuildTemplate + } else { + c.TemplateBody = clientTemplate + } return nil } -const clientTemplate = `{ +const ( + clientTemplate = `{ _config+:: { case_name: '{{ .CaseName }}', - image_name: 'hub.pingcap.net/qa/tipocket', + image_name: 'hub.pingcap.net/tipocket/tipocket', args+: { // k8s configurations // 'storage-class': 'local-storage', @@ -49,3 +55,16 @@ const clientTemplate = `{ }, } ` + clientIndividualBuildTemplate = `{ + _config+:: { + case_name: '{{ .CaseName }}', + image_name: 'hub.pingcap.net/tipocket/{{ .CaseName }}', + args+: { + // k8s configurations + // 'storage-class': 'local-storage', + }, + command: {}, + }, +} +` +) diff --git a/run/README.md b/run/README.md index aef2fffd7..1a8f796af 100644 --- a/run/README.md +++ b/run/README.md @@ -30,7 +30,7 @@ After fill on the testcase metadata, I should write a workflow file on [workflow { _config+:: { case_name: 'list_append', - image_name: 'hub.pingcap.net/qa/tipocket', + image_name: 'hub.pingcap.net/tipocket/tipocket', args+: { // k8s configurations // 'storage-class': 'local-storage', diff --git a/run/lib/config.libsonnet b/run/lib/config.libsonnet index f12c86427..8718d3de7 100644 --- a/run/lib/config.libsonnet +++ b/run/lib/config.libsonnet @@ -1,7 +1,7 @@ { _config:: { schedule: '0 0 * * *', - image_name: 'hub.pingcap.net/qa/tipocket', + image_name: 'hub.pingcap.net/tipocket/tipocket', args: { // cluster configurations hub: 'docker.io', diff --git a/run/workflow/bank.jsonnet b/run/workflow/bank.jsonnet index 5aaa20787..f6a14cdd0 100644 --- a/run/workflow/bank.jsonnet +++ b/run/workflow/bank.jsonnet @@ -1,7 +1,7 @@ { _config+:: { case_name: 'bank', - image_name: 'hub.pingcap.net/qa/tipocket', + image_name: 'hub.pingcap.net/tipocket/tipocket', args+: { // k8s configurations // 'storage-class': 'local-storage', diff --git a/run/workflow/bank2.jsonnet b/run/workflow/bank2.jsonnet index 74c99f90b..2aa72a136 100644 --- a/run/workflow/bank2.jsonnet +++ b/run/workflow/bank2.jsonnet @@ -1,7 +1,7 @@ { _config+:: { case_name: 'bank2', - image_name: 'hub.pingcap.net/qa/tipocket', + image_name: 'hub.pingcap.net/tipocket/tipocket', args+: { // k8s configurations // 'storage-class': 'local-storage', diff --git a/run/workflow/block-writer.jsonnet b/run/workflow/block-writer.jsonnet index b921df382..f168f76aa 100644 --- a/run/workflow/block-writer.jsonnet +++ b/run/workflow/block-writer.jsonnet @@ -1,7 +1,7 @@ { _config+:: { case_name: 'block-writer', - image_name: 'hub.pingcap.net/qa/tipocket', + image_name: 'hub.pingcap.net/tipocket/tipocket', args+: { // k8s configurations // 'storage-class': 'local-storage', diff --git a/run/workflow/cross-region.jsonnet b/run/workflow/cross-region.jsonnet index 381b84dff..085a4511f 100644 --- a/run/workflow/cross-region.jsonnet +++ b/run/workflow/cross-region.jsonnet @@ -1,7 +1,7 @@ { _config+:: { case_name: 'crossregion', - image_name: 'hub.pingcap.net/qa/tipocket', + image_name: 'hub.pingcap.net/tipocket:cross-region', args+: { // k8s configurations // 'storage-class': 'local-storage', diff --git a/run/workflow/example.jsonnet b/run/workflow/example.jsonnet index f9613b44d..2484722bb 100644 --- a/run/workflow/example.jsonnet +++ b/run/workflow/example.jsonnet @@ -1,7 +1,7 @@ { _config+:: { case_name: 'example', - image_name: 'hub.pingcap.net/qa/tipocket', + image_name: 'hub.pingcap.net/tipocket/tipocket', args+: { // k8s configurations // 'storage-class': 'local-storage', diff --git a/run/workflow/ledger.jsonnet b/run/workflow/ledger.jsonnet index b9d3d8f67..627cf64be 100644 --- a/run/workflow/ledger.jsonnet +++ b/run/workflow/ledger.jsonnet @@ -1,7 +1,7 @@ { _config+:: { case_name: 'ledger', - image_name: 'hub.pingcap.net/qa/tipocket', + image_name: 'hub.pingcap.net/tipocket/tipocket', args+: { // k8s configurations // 'storage-class': 'local-storage', diff --git a/run/workflow/list-append.jsonnet b/run/workflow/list-append.jsonnet index df947a219..a6fa6833c 100644 --- a/run/workflow/list-append.jsonnet +++ b/run/workflow/list-append.jsonnet @@ -1,7 +1,7 @@ { _config+:: { case_name: 'list-append', - image_name: 'hub.pingcap.net/qa/tipocket', + image_name: 'hub.pingcap.net/tipocket/tipocket', args+: { // k8s configurations // 'storage-class': 'local-storage', diff --git a/run/workflow/rawkv-linearizability.jsonnet b/run/workflow/rawkv-linearizability.jsonnet index 0c9206395..95b23cbca 100644 --- a/run/workflow/rawkv-linearizability.jsonnet +++ b/run/workflow/rawkv-linearizability.jsonnet @@ -1,7 +1,7 @@ { _config+:: { case_name: 'rawkv_linearizability', - image_name: 'hub.pingcap.net/qa/tipocket', + image_name: 'hub.pingcap.net/tipocket/tipocket', args+: { // k8s configurations // 'storage-class': 'local-storage', diff --git a/run/workflow/region-available.jsonnet b/run/workflow/region-available.jsonnet index 66dc032a7..b4000f816 100644 --- a/run/workflow/region-available.jsonnet +++ b/run/workflow/region-available.jsonnet @@ -1,7 +1,7 @@ { _config+:: { case_name: 'region_available', - image_name: 'hub.pingcap.net/qa/tipocket', + image_name: 'hub.pingcap.net/tipocket/tipocket', args+: { // k8s configurations // 'storage-class': 'local-storage', diff --git a/run/workflow/rw-register.jsonnet b/run/workflow/rw-register.jsonnet index 5ee060d19..42e09c852 100644 --- a/run/workflow/rw-register.jsonnet +++ b/run/workflow/rw-register.jsonnet @@ -1,7 +1,7 @@ { _config+:: { case_name: 'rw-register', - image_name: 'hub.pingcap.net/qa/tipocket', + image_name: 'hub.pingcap.net/tipocket/tipocket', args+: { // k8s configurations // 'storage-class': 'local-storage', diff --git a/run/workflow/sqllogic.jsonnet b/run/workflow/sqllogic.jsonnet index b87295d97..acc3e393b 100644 --- a/run/workflow/sqllogic.jsonnet +++ b/run/workflow/sqllogic.jsonnet @@ -1,7 +1,7 @@ { _config+:: { case_name: 'sqllogic', - image_name: 'hub.pingcap.net/qa/tipocket', + image_name: 'hub.pingcap.net/tipocket/tipocket', args+: { // k8s configurations // 'storage-class': 'local-storage', diff --git a/run/workflow/tpcc.jsonnet b/run/workflow/tpcc.jsonnet index 2f97a5709..5e1a21ce0 100644 --- a/run/workflow/tpcc.jsonnet +++ b/run/workflow/tpcc.jsonnet @@ -1,7 +1,7 @@ { _config+:: { case_name: 'tpcc', - image_name: 'hub.pingcap.net/qa/tipocket', + image_name: 'hub.pingcap.net/tipocket/tipocket', args+: { // k8s configurations // 'storage-class': 'local-storage', diff --git a/run/workflow/ttl.jsonnet b/run/workflow/ttl.jsonnet index cb155d21f..4baff27db 100644 --- a/run/workflow/ttl.jsonnet +++ b/run/workflow/ttl.jsonnet @@ -1,7 +1,7 @@ { _config+:: { case_name: 'ttl', - image_name: 'hub.pingcap.net/qa/tipocket', + image_name: 'hub.pingcap.net/tipocket/tipocket', args+: { // k8s configurations // 'storage-class': 'local-storage', diff --git a/run/workflow/txn-rand-pessimistic.jsonnet b/run/workflow/txn-rand-pessimistic.jsonnet index 73268667e..7d8561ee8 100644 --- a/run/workflow/txn-rand-pessimistic.jsonnet +++ b/run/workflow/txn-rand-pessimistic.jsonnet @@ -1,7 +1,7 @@ { _config+:: { case_name: 'txn_rand_pessimistic', - image_name: 'hub.pingcap.net/qa/tipocket', + image_name: 'hub.pingcap.net/tipocket/tipocket', args+: { // k8s configurations // 'storage-class': 'local-storage', diff --git a/run/workflow/vbank.jsonnet b/run/workflow/vbank.jsonnet index 34a4c77b3..3487c4331 100644 --- a/run/workflow/vbank.jsonnet +++ b/run/workflow/vbank.jsonnet @@ -1,7 +1,7 @@ { _config+:: { case_name: 'vbank', - image_name: 'hub.pingcap.net/qa/tipocket', + image_name: 'hub.pingcap.net/tipocket/tipocket', args+: { // k8s configurations // 'storage-class': 'local-storage', diff --git a/testcase/cross-region/Dockerfile b/testcase/cross-region/Dockerfile index 0d8e7793d..e1e3bb600 100644 --- a/testcase/cross-region/Dockerfile +++ b/testcase/cross-region/Dockerfile @@ -1,5 +1,23 @@ -FROM alpine:3.9 +# syntax = docker/dockerfile:1.0-experimental +FROM golang:alpine3.10 AS build_base -RUN apk update && apk add --no-cache wget bash sed -COPY ./bin/cross-region /bin/cross-region +RUN apk update && apk add --no-cache wget bash sed make git + +WORKDIR /src +COPY . . +COPY .git .git + +WORKDIR /src/testcase/cross-region + +RUN rm -rf /go/src/ +RUN --mount=type=cache,id=tipocket_go_pkg,target=/go/pkg \ + --mount=type=cache,id=tipocket_go_cache,target=/root/.cache/go-build \ + --mount=type=tmpfs,id=tipocket_go_src,target=/go/src/ make build + +FROM alpine:3.8 + +RUN mkdir -p /config && mkdir -p /resources +COPY --from=0 /src/config /config +COPY --from=0 /src/resources /resources +COPY --from=0 /src/testcase/cross-region/bin/cross-region /bin/cross-region ENTRYPOINT [ "/bin/cross-region" ]