diff --git a/.github/linters/.gitleaks.toml b/.github/linters/.gitleaks.toml new file mode 100644 index 0000000..150012a --- /dev/null +++ b/.github/linters/.gitleaks.toml @@ -0,0 +1,179 @@ + +title = "gitleaks config" + +[[rules]] + description = "AWS Access Key" + regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}''' + tags = ["key", "AWS"] + +[[rules]] + description = "AWS Secret Key" + regex = '''(?i)aws(.{0,20})?(?-i)['\"][0-9a-zA-Z\/+]{40}['\"]''' + tags = ["key", "AWS"] + +[[rules]] + description = "AWS MWS key" + regex = '''amzn\.mws\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}''' + tags = ["key", "AWS", "MWS"] + +[[rules]] + description = "Facebook Secret Key" + regex = '''(?i)(facebook|fb)(.{0,20})?(?-i)['\"][0-9a-f]{32}['\"]''' + tags = ["key", "Facebook"] + +[[rules]] + description = "Facebook Client ID" + regex = '''(?i)(facebook|fb)(.{0,20})?['\"][0-9]{13,17}['\"]''' + tags = ["key", "Facebook"] + +[[rules]] + description = "Twitter Secret Key" + regex = '''(?i)twitter(.{0,20})?['\"][0-9a-z]{35,44}['\"]''' + tags = ["key", "Twitter"] + +[[rules]] + description = "Twitter Client ID" + regex = '''(?i)twitter(.{0,20})?['\"][0-9a-z]{18,25}['\"]''' + tags = ["client", "Twitter"] + +[[rules]] + description = "Github Personal Access Token" + regex = '''ghp_[0-9a-zA-Z]{36}''' + tags = ["key", "Github"] +[[rules]] + description = "Github OAuth Access Token" + regex = '''gho_[0-9a-zA-Z]{36}''' + tags = ["key", "Github"] +[[rules]] + description = "Github App Token" + regex = '''(ghu|ghs)_[0-9a-zA-Z]{36}''' + tags = ["key", "Github"] +[[rules]] + description = "Github Refresh Token" + regex = '''ghr_[0-9a-zA-Z]{76}''' + tags = ["key", "Github"] + +[[rules]] + description = "LinkedIn Client ID" + regex = '''(?i)linkedin(.{0,20})?(?-i)[0-9a-z]{12}''' + tags = ["client", "LinkedIn"] + +[[rules]] + description = "LinkedIn Secret Key" + regex = '''(?i)linkedin(.{0,20})?[0-9a-z]{16}''' + tags = ["secret", "LinkedIn"] + +[[rules]] + description = "Slack" + regex = '''xox[baprs]-([0-9a-zA-Z]{10,48})?''' + tags = ["key", "Slack"] + +[[rules]] + description = "Asymmetric Private Key" + regex = '''-----BEGIN ((EC|PGP|DSA|RSA|OPENSSH) )?PRIVATE KEY( BLOCK)?-----''' + tags = ["key", "AsymmetricPrivateKey"] + +[[rules]] + description = "Google API key" + regex = '''AIza[0-9A-Za-z\\-_]{35}''' + tags = ["key", "Google"] + +[[rules]] + description = "Google (GCP) Service Account" + regex = '''"type": "service_account"''' + tags = ["key", "Google"] + +[[rules]] + description = "Heroku API key" + regex = '''(?i)heroku(.{0,20})?[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}''' + tags = ["key", "Heroku"] + +[[rules]] + description = "MailChimp API key" + regex = '''(?i)(mailchimp|mc)(.{0,20})?[0-9a-f]{32}-us[0-9]{1,2}''' + tags = ["key", "Mailchimp"] + +[[rules]] + description = "Mailgun API key" + regex = '''((?i)(mailgun|mg)(.{0,20})?)?key-[0-9a-z]{32}''' + tags = ["key", "Mailgun"] + +[[rules]] + description = "PayPal Braintree access token" + regex = '''access_token\$production\$[0-9a-z]{16}\$[0-9a-f]{32}''' + tags = ["key", "Paypal"] + +[[rules]] + description = "Picatic API key" + regex = '''sk_live_[0-9a-z]{32}''' + tags = ["key", "Picatic"] + +[[rules]] + description = "SendGrid API Key" + regex = '''SG\.[\w_]{16,32}\.[\w_]{16,64}''' + tags = ["key", "SendGrid"] + +[[rules]] + description = "Slack Webhook" + regex = '''https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8,12}/[a-zA-Z0-9_]{24}''' + tags = ["key", "slack"] + +[[rules]] + description = "Stripe API key" + regex = '''(?i)stripe(.{0,20})?[sr]k_live_[0-9a-zA-Z]{24}''' + tags = ["key", "Stripe"] + +[[rules]] + description = "Square access token" + regex = '''sq0atp-[0-9A-Za-z\-_]{22}''' + tags = ["key", "square"] + +[[rules]] + description = "Square OAuth secret" + regex = '''sq0csp-[0-9A-Za-z\\-_]{43}''' + tags = ["key", "square"] + +[[rules]] + description = "Twilio API key" + regex = '''(?i)twilio(.{0,20})?SK[0-9a-f]{32}''' + tags = ["key", "twilio"] + +[[rules]] + description = "Dynatrace ttoken" + regex = '''dt0[a-zA-Z]{1}[0-9]{2}\.[A-Z0-9]{24}\.[A-Z0-9]{64}''' + tags = ["key", "Dynatrace"] + +[[rules]] + description = "Shopify shared secret" + regex = '''shpss_[a-fA-F0-9]{32}''' + tags = ["key", "Shopify"] + +[[rules]] + description = "Shopify access token" + regex = '''shpat_[a-fA-F0-9]{32}''' + tags = ["key", "Shopify"] + +[[rules]] + description = "Shopify custom app access token" + regex = '''shpca_[a-fA-F0-9]{32}''' + tags = ["key", "Shopify"] + +[[rules]] + description = "Shopify private app access token" + regex = '''shppa_[a-fA-F0-9]{32}''' + tags = ["key", "Shopify"] + +[[rules]] + description = "PyPI upload token" + regex = '''pypi-AgEIcHlwaS5vcmc[A-Za-z0-9-_]{50,1000}''' + tags = ["key", "pypi"] + +[allowlist] + description = "Allowlisted files" + paths = [ + '''^\.?gitleaks.toml$''', + '''topo/node/srl/generate_certificate_success$''', # exclude dummy test file with random cert + '''(.*?)super-linter.log$''', # exclude linter logs which might contain past errored runs with keys/certs + '''(.*?)(png|jpg|gif|doc|docx|pdf|bin|xls|pyc|zip)$''', + '''(go.mod|go.sum)$''' + ] \ No newline at end of file diff --git a/.github/linters/.golangci.yml b/.github/linters/.golangci.yml new file mode 100644 index 0000000..dca2af2 --- /dev/null +++ b/.github/linters/.golangci.yml @@ -0,0 +1,53 @@ +--- +######################### +######################### +## Golang Linter rules ## +######################### +######################### + +# configure golangci-lint +# see https://github.com/golangci/golangci-lint/blob/master/.golangci.example.yml +run: + timeout: 10m +issues: + exclude-rules: + - path: _test\.go + linters: + - dupl + - gosec + - goconst + new: true +linters: + enable: + - gosec + - unconvert + - goconst + - goimports + - gofmt + - gocritic + - govet + - revive + - staticcheck + - unconvert + - unparam + - unused + - wastedassign + - whitespace +linters-settings: + errcheck: + # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`; + # default is false: such cases aren't reported by default. + check-blank: true + govet: + # report about shadowed variables + check-shadowing: false + maligned: + # print struct with more effective memory layout or not, false by default + suggest-new: true + gocritic: + disabled-checks: + - singleCaseSwitch + - appendAssign + revive: + ignore-generated-header: true + severity: warning diff --git a/.github/linters/.yaml-lint.yml b/.github/linters/.yaml-lint.yml new file mode 100644 index 0000000..e9ec8be --- /dev/null +++ b/.github/linters/.yaml-lint.yml @@ -0,0 +1,59 @@ +--- +########################################### +# These are the rules used for # +# linting all the yaml files in the stack # +# NOTE: # +# You can disable line with: # +# # yamllint disable-line # +########################################### +rules: + braces: + level: warning + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: 1 + max-spaces-inside-empty: 5 + brackets: + level: warning + min-spaces-inside: 0 + max-spaces-inside: 0 + min-spaces-inside-empty: 1 + max-spaces-inside-empty: 5 + colons: + level: warning + max-spaces-before: 0 + max-spaces-after: 1 + commas: + level: warning + max-spaces-before: 0 + min-spaces-after: 1 + max-spaces-after: 1 + comments: disable + comments-indentation: disable + document-end: disable + document-start: + level: warning + present: true + empty-lines: + level: warning + max: 2 + max-start: 0 + max-end: 0 + hyphens: + level: warning + max-spaces-after: 1 + indentation: + level: warning + spaces: consistent + indent-sequences: true + check-multi-line-strings: false + key-duplicates: enable + line-length: + level: warning + max: 120 + allow-non-breakable-words: true + allow-non-breakable-inline-mappings: true + new-line-at-end-of-file: disable + new-lines: + type: unix + trailing-spaces: disable diff --git a/.github/workflows/basic_go.yml b/.github/workflows/basic_go.yml index 5dda19f..6b6945e 100644 --- a/.github/workflows/basic_go.yml +++ b/.github/workflows/basic_go.yml @@ -1,16 +1,31 @@ -name: Common Go +name: Basic Go on: workflow_call: inputs: + go-versions: + type: string + # Arrays are currently not accepted: + # https://github.com/orgs/community/discussions/11692 + default: "['stable' , 'oldstable']" + staticcheck-version: + type: string + default: 'latest' static-analysis-excludes-regex: type: string # TODO(wenvous): gofmt is harder to work with since it always recursively # formats files instead of restricting itself to just listed packages. # See if there is a way to have the same behaviour without relying on the # tool. + tests-excludes-regex: + type: string + description: 'grep -E regex for excluding certain packages from testing' race-tests-excludes-regex: type: string + description: 'grep -E regex for excluding certain packages from race testing, this DOES NOT include packages excluded from regular testing' + coverage-excludes-regex: + type: string + description: 'grep -E regex for excluding certain packages from coverage reporting' skip-gofmt: type: boolean skip-govet: @@ -27,18 +42,18 @@ jobs: strategy: fail-fast: false matrix: - go: ['1.19', '1.20', '1.x'] + go: ${{ fromJson(inputs.go-versions) }} steps: + - name: Check out code + uses: actions/checkout@v3 + - name: Set up Go uses: actions/setup-go@v4 with: go-version: ${{ matrix.go }} id: go - - name: Check out code - uses: actions/checkout@v3 - - name: Get dependencies run: | go mod download @@ -47,7 +62,14 @@ jobs: run: go build -v ./... - name: Run Tests - run: go test -v ./... + run: | + if [ -z "${{ inputs.tests-excludes-regex }}" ]; then + echo "Running for all packages" + go test -v ./... + else + echo "Running for non-excluded packages" + go test -v $(go list ./... | grep -E -v "${{ inputs.tests-excludes-regex }}") + fi - name: Run race tests if: ${{ !inputs.skip-race-tests }} @@ -57,7 +79,7 @@ jobs: go test -race -v ./... else echo "Running for non-excluded packages" - go test -race -v $(go list ./... | egrep -v "${{ inputs.race-tests-excludes-regex }}") + go test -race -v $(go list ./... | grep -E -v "${{ inputs.race-tests-excludes-regex }}") fi static_analysis: @@ -65,15 +87,15 @@ jobs: runs-on: ubuntu-latest steps: + - name: Check out code + uses: actions/checkout@v3 + - name: Install Go uses: actions/setup-go@v4 with: - go-version: '1.x' + go-version: 'stable' id: go - - name: Check out code - uses: actions/checkout@v3 - - name: Get dependencies run: | go mod download @@ -91,7 +113,19 @@ jobs: go vet ./... else echo "Running for non-excluded packages" - go vet $(go list ./... | egrep -v "${{ inputs.static-analysis-excludes-regex }}") + go vet $(go list ./... | grep -E -v "${{ inputs.static-analysis-excludes-regex }}") + fi + + - name: revive + if: ${{ !cancelled() }} + run: | + go install github.com/mgechev/revive@latest + if [ -z "${{ inputs.static-analysis-excludes-regex }}" ]; then + echo "Running for all packages" + revive ./... + else + echo "Running for non-excluded packages" + revive $(go list ./... | grep -E -v ${{ inputs.static-analysis-excludes-regex }}) fi - name: Gofmt @@ -100,32 +134,63 @@ jobs: diff -u <(echo -n) <(gofmt -d -s .) - name: Run coverage - if: '!cancelled()' + # Always run unless manually cancelled. + if: ${{ !cancelled() }} run: | - go test -v -coverprofile=profile.cov ./... + if [ -z "${{ inputs.coverage-excludes-regex }}" ]; then + echo "Running for all packages" + go test -v -coverprofile=profile.cov ./... + else + echo "Running for non-excluded packages" + go test -v -coverprofile=profile.cov $(go list ./... | grep -E -v ${{ inputs.coverage-excludes-regex }}) + fi - name: Install goveralls - if: '!cancelled()' + if: ${{ !cancelled() }} run: go install github.com/mattn/goveralls@latest - name: Submit coverage - if: '!cancelled()' + if: ${{ !cancelled() }} env: COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: goveralls -coverprofile=profile.cov -service=github + staticcheck: + name: staticcheck + runs-on: ubuntu-latest + + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Install Go + uses: actions/setup-go@v4 + with: + go-version: 'stable' + id: go + - name: Install staticcheck - if: '!cancelled()' + if: ${{ !cancelled() }} run: | - go install honnef.co/go/tools/cmd/staticcheck@latest + go install honnef.co/go/tools/cmd/staticcheck@${{ inputs.staticcheck-version }} + + - name: Cache for staticcheck + if: ${{ !cancelled() && !inputs.skip-staticcheck }} + uses: actions/cache@v3 + with: + path: | + ${{ runner.temp }}/staticcheck + key: ${{ github.job }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('**/go.sum') }} + restore-keys: ${{ github.job }}-${{ steps.go.outputs.go-version }}-${{ hashFiles('**/go.sum') }} - name: Staticcheck if: ${{ !cancelled() && !inputs.skip-staticcheck }} run: | + export STATICCHECK_CACHE="${{ runner.temp }}/staticcheck" if [ -z "${{ inputs.static-analysis-excludes-regex }}" ]; then echo "Running for all packages" - staticcheck ./... + GOGC=30 staticcheck ./... else echo "Running for non-excluded packages" - staticcheck $(go list ./... | egrep -v ${{ inputs.static-analysis-excludes-regex }}) + GOGC=30 staticcheck $(go list ./... | grep -E -v ${{ inputs.static-analysis-excludes-regex }}) fi diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000..e900582 --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,133 @@ +name: Go + +on: + workflow_call: + inputs: + go-versions: + type: string + # Arrays are currently not accepted: + # https://github.com/orgs/community/discussions/11692 + default: "['stable', 'oldstable']" + tests-excludes-regex: + type: string + description: 'grep -E regex for excluding certain packages from testing' + race-tests-excludes-regex: + type: string + description: 'grep -E regex for excluding certain packages from race testing, this DOES NOT include packages excluded from regular testing' + coverage-excludes-regex: + type: string + description: 'grep -E regex for excluding certain packages from coverage reporting' + skip-race-tests: + type: boolean + +jobs: + build_and_test: + name: Build and Test + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + go: ${{ fromJson(inputs.go-versions) }} + + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: ${{ matrix.go }} + id: go + + - name: Get dependencies + run: | + go mod download + + - name: Go Mod should be tidy + run: | + go mod tidy + diff -u <(echo -n) <(git diff) + + - name: Build packages + run: go build -v ./... + + - name: Run coverage + run: | + if [ -z "${{ inputs.coverage-excludes-regex }}" ]; then + echo "Running for all packages" + go test -v -coverprofile=profile.cov ./... + else + echo "Running for non-excluded packages" + go test -v -coverprofile=profile.cov $(go list ./... | grep -E -v ${{ inputs.coverage-excludes-regex }}) + fi + + - name: Install goveralls + if: ${{ matrix.go == fromJson(inputs.go-versions)[0] }} + run: go install github.com/mattn/goveralls@latest + + - name: Submit coverage + if: ${{ matrix.go == fromJson(inputs.go-versions)[0] }} + env: + COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: goveralls -coverprofile=profile.cov -service=github + + - name: Run Tests + run: | + if [ -z "${{ inputs.tests-excludes-regex }}" ]; then + echo "Running for all packages" + go test -v ./... + else + echo "Running for non-excluded packages" + go test -v $(go list ./... | grep -E -v "${{ inputs.tests-excludes-regex }}") + fi + + - name: Run race tests + if: ${{ !inputs.skip-race-tests }} + run: | + if [ -z "${{ inputs.race-tests-excludes-regex }}" ]; then + echo "Running for all packages" + go test -race -v ./... + else + echo "Running for non-excluded packages" + go test -race -v $(go list ./... | grep -E -v "${{ inputs.race-tests-excludes-regex }}") + fi + + golangci: + name: lint + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v3 + with: + fetch-depth: '0' + + - name: Install Go + uses: actions/setup-go@v4 + with: + go-version: 'stable' + + - name: golangci-lint + uses: golangci/golangci-lint-action@v3 + with: + # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version + version: latest + + # Optional: working directory, useful for monorepos + # working-directory: somedir + + # Optional: golangci-lint command line arguments. + # args: --issues-exit-code=0 + args: --config=.github/linters/.golangci.yml + + # Optional: show only new issues if it's a pull request. The default value is `false`. + # only-new-issues: true + + # Optional: if set to true then the all caching functionality will be complete disabled, + # takes precedence over all other caching options. + # skip-cache: true + + # Optional: if set to true then the action don't cache or restore ~/go/pkg. + # skip-pkg-cache: true + + # Optional: if set to true then the action don't cache or restore ~/.cache/go-build. + # skip-build-cache: true diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml new file mode 100644 index 0000000..8b8a2aa --- /dev/null +++ b/.github/workflows/linter.yml @@ -0,0 +1,31 @@ +name: Linter + +on: + push: + branches: [ main ] + pull_request: + workflow_call: + +jobs: + lint: + name: Lint Code Base + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Lint Code Base + uses: super-linter/super-linter@v5 + env: + VALIDATE_ALL_CODEBASE: false + VALIDATE_GO: false + VALIDATE_HTML: false + VALIDATE_KUBERNETES_KUBEVAL: false + VALIDATE_JSCPD: false + # GitHub actions often contain $(go list ./...) as part of commands + GITHUB_ACTIONS_COMMAND_ARGS: -ignore SC2046 + IGNORE_GENERATED_FILES: true + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/README.md b/README.md index cbb2063..0cfc70a 100644 --- a/README.md +++ b/README.md @@ -1 +1,70 @@ # Common CI workflows for OpenConfig Projects + +## Standard Usage + +Copy the following into your repository's `.github/workflows/go.yml`: + +```yaml +name: Go + +on: + push: + branches: [ master ] + pull_request: + schedule: + - cron: "0 0 * * *" + +jobs: + go: + uses: openconfig/common-ci/.github/workflows/go.yml@v1.0.0 + + linter: + uses: openconfig/common-ci/.github/workflows/linter.yml@v1.0.0 +``` + +Then copy the sample linter configurations folder into your repository: + +```bash +cp -r common-ci/.github/linters YOUR-REPO/.github/linters +``` + +At this point, you may wish to change the configuration of these linters to suit +your own repository. + +## Basic Usage + +If you do not want to maintain linter configuration, then you may use the basic +workflow, which only involves copying the following into your repository's +`.github/workflows/go.yml`: + +```yaml +name: Go + +on: + push: + branches: [ master ] + pull_request: + schedule: + - cron: "0 0 * * *" + +jobs: + go: + uses: openconfig/common-ci/.github/workflows/basic_go.yml@v1.0.0 +``` + +## Flags + +Some flags are supported for customizing the CI workflow, for example, + +```yaml +jobs: + go: + uses: openconfig/common-ci/.github/workflows/basic_go.yml@v1.0.0 + with: + static-analysis-excludes-regex: exampleoc + skip-gofmt: true + skip-staticcheck: true +``` + +For all supported flags, see the corresponding reusable workflow definitions +under [.github/workflows](.github/workflows).