rough-out new BackfillSync class structure

.dockerignore

@@ -67,5 +67,6 @@ docs/site
 **/dist
 *.log
 **/*.log
+**/*.tsbuildinfo
 .idea
 .tmp

.env.test

@@ -1,10 +1,10 @@
 # We use these images during sim and e2e tests
 # This is the last version which supports pre/post merge chains in the same network
 # All newer versions only work with post merge chains
-GETH_DOCKER_IMAGE=ethereum/client-go:v1.13.14
+GETH_DOCKER_IMAGE=ethereum/client-go:v1.16.2
 # Use either image or local binary for the testing
 GETH_BINARY_DIR=
-LIGHTHOUSE_DOCKER_IMAGE=sigp/lighthouse:v7.0.1
+LIGHTHOUSE_DOCKER_IMAGE=ethpandaops/lighthouse:unstable-d235f2c
 
 # We can't upgrade nethermind further due to genesis hash mismatch with the geth
 # https://github.com/NethermindEth/nethermind/issues/6683

.github/workflows/publish-nextfork.yml

@@ -0,0 +1,152 @@
+name: Publish nextfork release
+
+# only one per github sha can be run
+concurrency:
+  group: cd-publish-nextfork
+
+on:
+  push:
+    branches:
+      - peerDAS  # Nextfork branch
+
+env:
+  NEXT_FORK: peerDAS
+
+jobs:
+  npm:
+    name: Publish to NPM Registry
+    runs-on: buildjet-4vcpu-ubuntu-2204
+    steps:
+      # <common-build> - Uses YAML anchors in the future
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          registry-url: "https://registry.npmjs.org"
+          check-latest: true
+          cache: yarn
+      - name: Node.js version
+        id: node
+        run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT
+      - name: Restore dependencies
+        uses: actions/cache@master
+        id: cache-deps
+        with:
+          path: |
+            node_modules
+            packages/*/node_modules
+          key: ${{ runner.os }}-${{ steps.node.outputs.v8CppApiVersion }}-${{ hashFiles('**/yarn.lock', '**/package.json') }}
+      - name: Install & build
+        if: steps.cache-deps.outputs.cache-hit != 'true'
+        run: yarn install --frozen-lockfile && yarn build
+      - name: Build
+        run: yarn build
+        if: steps.cache-deps.outputs.cache-hit == 'true'
+      # </common-build>
+      - name: Get version
+        id: version
+        run: |
+          PACKAGE_VERSION=$(node -p "require('./packages/cli/package.json').version")
+          NEXT_VERSION=$(npx --yes semver --increment minor $PACKAGE_VERSION)
+          export VERSION=${NEXT_VERSION}-${NEXT_FORK}.${GITHUB_SHA:0:10}
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+          echo PACKAGE_VERSION $PACKAGE_VERSION GITHUB_SHA $GITHUB_SHA VERSION $VERSION
+
+      - name: Change and commit version
+        # Write version before publishing so it's picked up by `lerna publish from-package`.
+        # It must also be committed to ensure a clean git tree, otherwise `lerna publish` errors.
+        # This "temp" commit doesn't change the actually release commit which is captured above.
+        # git-data is also correct, since it's generated at build time, before `lerna version` run.
+        run: |
+          node_modules/.bin/lerna version ${{ steps.version.outputs.version }} \
+          --force-publish \
+          --exact \
+          --yes \
+          --no-git-tag-version
+
+          git config user.name 'temp'
+          git config user.email '[email protected]'
+          git commit -am "${{ steps.version.outputs.version }}"
+
+      - name: Publish to npm registry
+        # Note: before https://github.com/ChainSafe/lodestar/commit/28e2c74cf0f1bede8b09c8c9fec26f54b367e3fd
+        # We used `lerna publish --canary` option. However, since we now publish must version on branches,
+        # i.e. v0.35.x branch, lerna fails to detect the latest version and publishes canary versions as
+        # `0.34.0-dev.173+28e2c74cf0` instead of `0.36.0-dev.4+28e2c74cf0`, which creates confusion.
+        #
+        # --no-git-reset:
+        #   Do not delete code version artifacts so the next step can pick the version
+        #
+        # --dist-tag next:
+        #   Make this dev version installable with `@next`
+        #
+        # --preid dev:
+        #   Tag version with `dev` instead of `alpha`
+        #
+        # --force-publish:
+        #   lerna doesn't want to publish anything otherwise - "lerna success No changed packages
+        #   to publish"
+        # --exact
+        #   lerna will link the dependencies of monorepo packages without ^ operator as npm
+        #   is apparently bad at resolving ^ dependencies of the canary versions. For e.g
+        #   @chainsafe/lodestar@^0.34.0-dev.4 resolves to => 0.34.0
+        #
+        # NOTE: Using --preid dev.$(git rev-parse --short=7 HEAD) results in `0.24.3-dev.3ddb91d.0+3ddb91d`
+        run: |
+          node_modules/.bin/lerna publish from-package \
+          --yes \
+          --no-verify-access \
+          --dist-tag next \
+          --no-git-reset \
+          --force-publish \
+          --exact
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+    outputs:
+      version: ${{ steps.version.outputs.version }}
+
+  docker:
+    name: Publish to Docker Hub
+    runs-on: buildjet-4vcpu-ubuntu-2204
+    needs: npm
+    steps:
+      - uses: actions/checkout@v4
+      # https://github.com/docker/setup-qemu-action
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      # https://github.com/docker/setup-buildx-action
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push lodestar
+        run: >
+          docker buildx build . --push
+          --tag chainsafe/lodestar:nextfork
+          --platform linux/amd64,linux/arm64
+          --build-arg COMMIT=$(git rev-parse HEAD)
+
+      - run: docker run chainsafe/lodestar:nextfork --help
+      # Display history to know byte size of each layer
+      # Image is available only because of the previous `docker run` command
+      - run: docker image history chainsafe/lodestar:nextfork
+
+      - name: Build and push custom Grafana
+        run: >
+          docker buildx build ./docker/grafana/ --push
+          --file ./docker/grafana/Dockerfile
+          --build-context dashboards=./dashboards
+          --tag chainsafe/lodestar-grafana:nextfork
+          --platform linux/amd64,linux/arm64
+
+      - name: Build and push custom Prometheus
+        run: >
+          docker buildx build ./docker/prometheus/ --push
+          --file ./docker/prometheus/Dockerfile
+          --tag chainsafe/lodestar-prometheus:nextfork
+          --platform linux/amd64,linux/arm64

.github/workflows/test.yml

@@ -53,7 +53,7 @@ jobs:
         run: scripts/assert_no_yarn_warnings.sh
 
       - name: Lint yarn lockfile
-        run: npx lockfile-lint --path yarn.lock --allowed-hosts npm yarn --validate-https
+        run: npx lockfile-lint --path yarn.lock --allowed-hosts npm yarn github.com --allowed-schemes "https:" "git+https:"
 
       - name: Lint Code
         run: yarn lint

CONTRIBUTING.md

@@ -2,6 +2,57 @@
 
 Thanks for your interest in contributing to Lodestar. It's people like you that push the Ethereum ecosystem forward.
 
+## New Contributor?
+
+Unsure where to begin contributing to Lodestar? Here are some ideas!
+
+- :mag_right: Look through our [good first issues](https://github.com/ChainSafe/lodestar/issues?q=state%3Aopen%20label%3A%22good%20first%20issue%22).
+- :speech_balloon: Join our [Discord chat](https://discord.gg/aMxzVcr)!
+  [![Discord](https://img.shields.io/discord/593655374469660673.svg?label=Discord&logo=discord)](https://discord.gg/aMxzVcr)
+
+### AI Assistance Notice
+
+> [!IMPORTANT]
+>
+> If you are using **any kind of AI assistance** to contribute to Lodestar, **this must be disclosed in the pull request** to maintainers
+> along with the extent to which AI assistance was used (e.g. docs only vs. code generation).
+> If PR responses are being generated by an AI, disclose that as well.
+> As a small exception, trivial tab-completion doesn't need to be disclosed, so long as it is limited to single keywords or short phrases.
+
+An example disclosure:
+
+> This PR was written primarily by Claude Code.
+> I consulted Claude Code to understand the codebase, but the solution was fully authored manually by myself.
+
+Failure to disclose this is first and foremost, rude to the human maintainers on the other end of the pull request and it makes it difficult to determine how much scrutiny to apply to the contribution.
+
+In a perfect world, AI assistance would produce equal or higher quality of work than any human. This is currently not the case and most of the time is generating AI slop which is noticeable in patterns and lack of context to solving the problem. We say this as heavy users of AI to help maintain this project (with careful supervision).
+
+When using AI assistance, we expect contributors to understand the code that is produced and be able to answer critical questions about what was submitted. It isn't a maintainer's job to review a PR so broken that it requires significant rework to be acceptable.
+
+Most of all, it is disrespectful to our time. Please be respectful to our maintainers and **disclose AI assistance.** Using AI without a thorough review and understanding of the output does not enhance your reputation as a valuable contributor or increase the likelihood of future opportunities on this project.
+
+## Reporting A Bug?
+
+- :spiral_notepad: [Create a new issue!](https://github.com/ChainSafe/lodestar/issues/new/choose) Select the type of issue that best fits, and please fill out as much of the information as you can.
+
+## Contribution Process
+
+1. Make sure you're familiar with our contribution guidelines _(this document)_!
+2. Create your [own fork](https://github.com/ChainSafe/lodestar/fork) of this repository.
+3. Make your changes in your local fork.
+4. If you've made a code change, make sure to lint and test your changes (`yarn lint`, `yarn check-types` and `yarn test:unit`).
+5. Make an open pull request when you're ready for it to be reviewed. We review PRs on a regular basis. See Pull request etiquette for more information.
+
+> [!IMPORTANT]
+> Please note that trivial, non-code contributions such as spelling, grammar, typos, corrections, comments and link fixes are not acceptable pull requests.
+> Although we appreciate the effort to fix these valid concerns, it is not practical for us to run our CI systems to accommodate minor external contributions which generate minimal value for the purpose of contribution/airdrop farming.
+> It would be appreciated for you to open up an issue instead for our team to aggregate these types of contributions into a batch commit.
+
+## Lodestar Monorepo
+
+We currently host the majority of lodestar packages and support packages in [this repository](https://github.com/ChainSafe/lodestar) as a [monorepo](https://en.wikipedia.org/wiki/Monorepo). We're using [Lerna](https://lerna.js.org/) to manage the packages. See [packages/](https://github.com/ChainSafe/lodestar/tree/unstable/packages) for a list of packages hosted in this repository.
+
 ## Prerequisites
 
 - :gear: [NodeJS](https://nodejs.org/) (LTS)
@@ -114,29 +165,6 @@ Run a local beacon with `--metrics` enabled. Then start Prometheus + Grafana wit
 ./docker/docker-compose.local_dev.sh
 ```
 
-## First Time Contributor?
-
-Unsure where to begin contributing to Lodestar? Here are some ideas!
-
-- :pencil2: See any typos? See any verbiage that should be changed or updated? Go for it! Github makes it easy to make contributions right from the browser.
-- :mag_right: Look through our [outstanding unassigned issues](https://github.com/ChainSafe/lodestar/issues?q=is%3Aopen+is%3Aissue+no%3Aassignee). (Hint: look for issues labeled `good first issue` or `help-wanted`!)
-- :speech_balloon: Join our [Discord chat](https://discord.gg/aMxzVcr)!
-  [![Discord](https://img.shields.io/discord/593655374469660673.svg?label=Discord&logo=discord)](https://discord.gg/aMxzVcr)
-
-## Reporting A Bug?
-
-- :spiral_notepad: [Create a new issue!](https://github.com/ChainSafe/lodestar/issues/new/choose) Select the type of issue that best fits, and please fill out as much of the information as you can.
-
-## Contribution Process
-
-1. Make sure you're familiar with our contribution guidelines _(this document)_!
-2. Create your [own fork](https://github.com/ChainSafe/lodestar/fork) of this repository.
-3. Make your changes in your local fork.
-4. If you've made a code change, make sure to lint and test your changes (`yarn lint` and `yarn test:unit`).
-5. Make an open pull request when you're ready for it to be reviewed. We review PRs on a regular basis. See Pull request etiquette for more information.
-
-Please note that trivial, non-code contributions such as spelling, grammar, typos, corrections, comments and link fixes are not acceptable pull requests. Although we appreciate the effort to fix these valid concerns, it is not practical for us to run our CI systems to accommodate minor external contributions which generate minimal value for the purpose of contribution/airdrop farming. It would be appreciated for you to open up an issue instead for our team to aggregate these types of contributions into a batch commit.
-
 ## Github Style Guide
 
 **Branch Naming**
@@ -165,11 +193,7 @@ For example:
 - If your pull request is fixable and needs additional changes or commits within a short period of time, switch your pull request into a draft until it's ready.
 - Otherwise, close your pull request and [create a new issue instead.](https://github.com/ChainSafe/lodestar/issues/new/choose)
 
-## Lodestar Monorepo
-
-We're currently experimenting with hosting the majority of lodestar packages and support packages in this repository as a [monorepo](https://en.wikipedia.org/wiki/Monorepo). We're using [Lerna](https://lerna.js.org/) to manage the packages. See [packages/](https://github.com/ChainSafe/lodestar/tree/unstable/packages) for a list of packages hosted in this repository.
-
-## Style Guide
+## Lodestar Code Style Guide
 
 - Lodestar has migrated to using ES modules.
 - Many module class constructors have the following signature: `(options, dependencies)`
@@ -202,7 +226,7 @@ We're currently experimenting with hosting the majority of lodestar packages and
   - Metric code variables on the other hand should not be suffixed, i.e. `Sec`-suffix should be omitted
   - Time-based metrics must use seconds as the unit
 
-## Tests style guide
+## Tests Style Guide
 
 Test must not depend on external live resources, such that running tests for a commit must be deterministic:
 
@@ -266,7 +290,7 @@ If the PR introduces large-scale changes, affecting multiple areas of the codeba
 - Some corner cases can be observed while testing on feature group. If any such issue is identified which was not brainstormed earlier must be bring to attention of the team.
 - If and when possible try to release big features under opt-in feature flags.
 
-## Logging policy
+## Logging Policy
 
 ### Logging Levels
 
@@ -277,7 +301,7 @@ Contributors must choose the log level carefully to ensure a consistent experien
 - `info`: General sporadic informational about the node's state. Examples include initialization messages, infrequent periodic status updates, or high-level progress reports.
 - `debug`: Detailed diagnostic information that can help developers or users troubleshoot specific issues. Examples include individual request logs for every REST API, networking interactions, or internal components status changes. Alias to `verbose`.
 
-### Logging guidelines
+### Logging Guidelines
 
 - Avoid excessive logging. Log messages should be clear and concise, providing enough information to understand the context and severity of the issue.
 - Do not log sensitive data, such as private keys, user credentials, or personal information.
@@ -345,7 +369,3 @@ A simple indicator of issue prioritization. It mainly applies to issues.
 ###### `spec.*` Ethereum Consensus Spec Version Target
 
 Issues that target a specific version of the Ethereum consensus spec, shall be tagged accordingly.
-
-## Community
-
-Come chat with us on [Discord](https://discord.gg/aMxzVcr) and join our public weekly planning meetings!

Dockerfile

@@ -4,7 +4,7 @@
 FROM --platform=${BUILDPLATFORM:-amd64} node:22-slim AS build_src
 ARG COMMIT
 WORKDIR /usr/app
-RUN apt-get update && apt-get install -y g++ make python3 python3-setuptools && apt-get clean && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y git g++ make python3 python3-setuptools && apt-get clean && rm -rf /var/lib/apt/lists/*
 
 COPY . .
 
@@ -23,7 +23,7 @@ RUN cd packages/cli && GIT_COMMIT=${COMMIT} yarn write-git-data
 # Note: This step is redundant for the host arch
 FROM node:22-slim AS build_deps
 WORKDIR /usr/app
-RUN apt-get update && apt-get install -y g++ make python3 python3-setuptools && apt-get clean && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y git g++ make python3 python3-setuptools && apt-get clean && rm -rf /var/lib/apt/lists/*
 
 COPY --from=build_src /usr/app .