refactor: cache manifest (#41)

* refactor: cache manifest to disk between compilations

* fix: fixing some issues with manifest cache on disk

Test plan: Made an ape project with one dep and one vyper contract. Ran
`ape console`. First `project.contracts` lookup caused compilation.
Subsequent lookups used the cache. Changing contract contents triggered
a re-compile of only the changed contract. Adding a new contract
triggered a compilation of just the one new file. Deleting a file caused
the contract to disappear from `project.contracts`.

* fix: use `with` to avoid leaking open file in eth_pm compiler

Saw this warning while compiling JSON eth_pm packages.
    $ ( rm -rf .build/ ; ape console )
    In [1]: contracts = project.contracts
    INFO: Compiling 'contracts/contract.vy'
    INFO: Compiling 'contracts/escrow.json'
    /src/ape/src/ape_pm/compiler.py:23: ResourceWarning: unclosed file <_io.TextIOWrapper name='/src/ape/example-ignore/contracts/escrow.json' mode='r' encoding='UTF-8'>
      data = json.load(path.open())
    ResourceWarning: Enable tracemalloc to get the object allocation traceback

* refactor: remove unused `ProjectManager.manifest` property

This property is unused and when I was experimenting in the console its
existence confused me. It's using `self.sources` to build the
`PackageManifest`, which is `List[Path]` when it should be `Dict[str,
Source]`. I also think it's confusing to have a `manifest` and
`cached_manifest` property. I'll add this property back if its needed.

* fix: give Source a `compute_checksum` method and use it

Needed this to fix these mypy errors:

    src/ape/managers/project.py:116: error: Item "None" of "Optional[Checksum]" has no attribute "algorithm"
    src/ape/managers/project.py:118: error: Item "None" of "Optional[Checksum]" has no attribute "hash"

* refactor: code review feedback

* refactor: detect deleted ethpm JSON files via new sourcePath attribute

* refactor: isort a few files flagged by CI

* refactor: code review feedback

Instead of duplicating checksum logic in `Source.compute_checksum`, use
the `compute_checksum` util function.

Fix isort issues (older version of isort was classifying requests as a
first party package).

Co-authored-by: Just some guy <[email protected]>

.pre-commit-config.yaml

@@ -10,7 +10,7 @@ repos:
     -   id: seed-isort-config
 
 -   repo: https://github.com/pre-commit/mirrors-isort
-    rev: v4.3.21
+    rev: v5.8.0
     hooks:
       - id: isort
 

pyproject.toml

@@ -22,7 +22,7 @@ markers = "fuzzing: Run Hypothesis fuzz test suite"
 line_length = 100
 force_grid_wrap = 0
 include_trailing_comma = true
-known_third_party = ["hypothesis"]
-known_first_party = ["ape_accounts"]
+known_third_party = ["hypothesis", "eth_account", "pytest", "dataclassy"]
+known_first_party = ["ape_accounts", "ape"]
 multi_line_output = 3
 use_parentheses = true

src/ape/api/compiler.py

@@ -1,6 +1,6 @@
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import List
+from typing import List, Set
 
 from ape.types import ContractType
 
@@ -11,6 +11,10 @@ class CompilerAPI(ABC):
     def name(self) -> str:
         ...
 
+    @abstractmethod
+    def get_versions(self, all_paths: List[Path]) -> Set[str]:
+        ...
+
     @abstractmethod
     def compile(self, contract_filepaths: List[Path]) -> List[ContractType]:
         """

src/ape/managers/config.py

@@ -1,6 +1,6 @@
 import json
 from pathlib import Path
-from typing import Dict
+from typing import Dict, List
 
 from dataclassy import dataclass
 
@@ -16,22 +16,28 @@ class ConfigManager:
     DATA_FOLDER: Path
     REQUEST_HEADER: Dict
     PROJECT_FOLDER: Path
+    name: str = ""
+    version: str = ""
+    dependencies: List[str] = []
     plugin_manager: PluginManager
     _plugin_configs: Dict[str, ConfigItem] = dict()
 
     def __init__(self):
         config_file = self.PROJECT_FOLDER / CONFIG_FILE_NAME
+
         if config_file.exists():
             user_config = load_config(config_file)
         else:
             user_config = {}
 
+        # Top level config items
+        self.name = user_config.pop("name", "")
+        self.version = user_config.pop("version", "")
+        self.dependencies = user_config.pop("dependencies", [])
+
         for plugin_name, config_class in self.plugin_manager.config_class:
-            if plugin_name in user_config:
-                user_override = user_config[plugin_name]
-                del user_config[plugin_name]  # For checking if all config was processed
-            else:
-                user_override = {}
+            # NOTE: `dict.pop()` is used for checking if all config was processed
+            user_override = user_config.pop(plugin_name, {})
 
             # NOTE: Will raise if improperly provided keys
             config = config_class(**user_override)

src/ape/managers/project.py

@@ -1,9 +1,12 @@
+import json
 from pathlib import Path
-from typing import Dict, List
+from typing import Dict, List, Optional
 
+import requests
 from dataclassy import dataclass
 
-from ape.types import ContractType  # Compiler, PackageManifest, PackageMeta
+from ape.types import Checksum, Compiler, ContractType, PackageManifest, Source  # PackageMeta
+from ape.utils import compute_checksum
 
 from .compilers import CompilerManager
 from .config import ConfigManager
@@ -15,20 +18,51 @@ class ProjectManager:
     config: ConfigManager
     compilers: CompilerManager
 
-    depedendencies: Dict[str, "ProjectManager"] = dict()
+    dependencies: Dict[str, PackageManifest] = dict()
 
     def __init__(self):
-        pass  # Look for depedencies from config
+        if isinstance(self.path, str):
+            self.path = Path(self.path)
+
+        self.dependencies = {
+            manifest.name: manifest
+            for manifest in map(self._extract_manifest, self.config.dependencies)
+        }
+
+    def _extract_manifest(self, manifest_uri: str) -> PackageManifest:
+        manifest_dict = requests.get(manifest_uri).json()
+        # TODO: Handle non-manifest URLs e.g. Ape/Brownie projects, Hardhat/Truffle projects, etc.
+        if "name" not in manifest_dict:
+            raise  # Dependencies must have a name!
+        return PackageManifest.from_dict(manifest_dict)
 
     def __str__(self) -> str:
         return f'Project("{self.path}")'
 
+    @property
     def _cache_folder(self) -> Path:
         folder = self.path / ".build"
         # NOTE: If we use the cache folder, we expect it to exist
         folder.mkdir(exist_ok=True)
         return folder
 
+    @property
+    def manifest_cachefile(self) -> Path:
+        file_name = self.config.name or "__local__"
+        return self._cache_folder / (file_name + ".json")
+
+    @property
+    def cached_manifest(self) -> Optional[PackageManifest]:
+        manifest_file = self.manifest_cachefile
+        if manifest_file.exists():
+            manifest_json = json.loads(manifest_file.read_text())
+            if "manifest" not in manifest_json:
+                raise  # Corrupted Manifest
+            return PackageManifest.from_dict(manifest_json)
+
+        else:
+            return None
+
     # NOTE: Using these paths should handle the case when the folder doesn't exist
     @property
     def _contracts_folder(self) -> Path:
@@ -42,41 +76,114 @@ def sources(self) -> List[Path]:
 
         return files
 
+    def load_contracts(self, use_cache: bool = True) -> Dict[str, ContractType]:
+        # Load a cached or clean manifest (to use for caching)
+        manifest = use_cache and self.cached_manifest or PackageManifest()
+        cached_sources = manifest.sources or {}
+        cached_contract_types = manifest.contractTypes or {}
+
+        # If a file is deleted from `self.sources` but is in `cached_sources`,
+        # remove its corresponding `contract_types` by using
+        # `ContractType.sourceId` and `ContractType.sourcePath`
+        deleted_sources = cached_sources.keys() - set(map(str, self.sources))
+        contract_types = {}
+        for name, ct in cached_contract_types.items():
+            if ct.sourcePath and str(ct.sourcePath) in deleted_sources:
+                pass  # the ethpm JSON file containing this contract was deleted
+            elif ct.sourceId in deleted_sources:
+                pass  # this contract's source code file was deleted
+            else:
+                contract_types[name] = ct
+
+        def file_needs_compiling(source: Path) -> bool:
+            path = str(source)
+            # New file added?
+            if path not in cached_sources:
+                return True
+
+            # Recalculate checksum if it doesn't exist yet
+            cached = cached_sources[path]
+            cached.compute_checksum(algorithm="md5")
+            assert cached.checksum  # to tell mypy this can't be None
+
+            # File contents changed in source code folder?
+            checksum = compute_checksum(
+                source.read_bytes(),
+                algorithm=cached.checksum.algorithm,
+            )
+            return checksum != cached.checksum.hash
+
+        # NOTE: filter by checksum, etc., and compile what's needed
+        #       to bring our cached manifest up-to-date
+        needs_compiling = filter(file_needs_compiling, self.sources)
+        contract_types.update(self.compilers.compile(list(needs_compiling)))
+
+        # Update cached contract types & source code entries in cached manifest
+        manifest.contractTypes = contract_types
+        cached_sources = {
+            str(source): Source(  # type: ignore
+                checksum=Checksum(  # type: ignore
+                    algorithm="md5", hash=compute_checksum(source.read_bytes())
+                ),
+                urls=[],
+            )
+            for source in self.sources
+        }
+        manifest.sources = cached_sources
+
+        # NOTE: Cache the updated manifest to disk (so `self.cached_manifest` reads next time)
+        self.manifest_cachefile.write_text(json.dumps(manifest.to_dict()))
+
+        return contract_types
+
     @property
     def contracts(self) -> Dict[str, ContractType]:
-        return self.compilers.compile(self.sources)
+        return self.load_contracts()
 
+    def __getattr__(self, attr_name: str):
+        contracts = self.load_contracts()
+        if attr_name in contracts:
+            return contracts[attr_name]
+        elif attr_name in self.dependencies:
+            return self.dependencies[attr_name]
+        else:
+            raise AttributeError(f"{self.__class__.__name__} has no attribute '{attr_name}'")
+
+    @property
     def _interfaces_folder(self) -> Path:
         return self.path / "interfaces"
 
+    @property
     def _scripts_folder(self) -> Path:
         return self.path / "scripts"
 
+    @property
     def _tests_folder(self) -> Path:
         return self.path / "tests"
 
     # TODO: Make this work for generating and caching the manifest file
+    @property
+    def compiler_data(self) -> List[Compiler]:
+        compilers = []
+
+        for extension, compiler in self.compilers.registered_compilers.items():
+            for version in compiler.get_versions(
+                [p for p in self.sources if p.suffix == extension]
+            ):
+                compilers.append(Compiler(compiler.name, version))  # type: ignore
+
+        return compilers
 
     # @property
     # def meta(self) -> PackageMeta:
     #     return PackageMeta(**self.config.get_config("ethpm").serialize())
 
-    # @property
-    # def manifest(self) -> PackageManifest:
-    #     return PackageManifest(
-    #         name=self.config.name,
-    #         version=self.config.version,
-    #         meta=self.meta,
-    #         sources=self.sources,
-    #         contractTypes=list(self.contracts.values()),
-    #         compilers=list(
-    #             Compiler(c.name, c.version)  # type: ignore
-    #             for c in self.compilers.registered_compilers.values()
-    #         ),
-    #     )
-
     # def publish_manifest(self):
     #     manifest = self.manifest.to_dict()  # noqa: F841
-    #     # TODO: Clean up manifest
+    #     if not manifest["name"]:
+    #         raise  # Need name to release manifest
+    #     if not manifest["version"]:
+    #         raise  # Need version to release manifest
+    #     # TODO: Clean up manifest and minify it
     #     # TODO: Publish sources to IPFS and replace with CIDs
     #     # TODO: Publish to IPFS

src/ape/types/__init__.py

@@ -1,10 +1,12 @@
-from .contract import Bytecode, Compiler, ContractType
+from .contract import Bytecode, Checksum, Compiler, ContractType, Source
 from .manifest import PackageManifest, PackageMeta
 
 __all__ = [
     "Bytecode",
+    "Checksum",
     "Compiler",
     "ContractType",
     "PackageManifest",
     "PackageMeta",
+    "Source",
 ]

src/ape/types/contract.py

@@ -1,7 +1,10 @@
 import urllib.request
 from copy import deepcopy
+from pathlib import Path
 from typing import Dict, List, Optional
 
+from ape.utils import compute_checksum
+
 from .abstract import SerializableType, update_list_params, update_params
 
 
@@ -56,6 +59,7 @@ class Compiler(SerializableType):
 class ContractType(SerializableType):
     contractName: str
     sourceId: Optional[str] = None
+    sourcePath: Optional[Path] = None
     deploymentBytecode: Optional[Bytecode] = None
     runtimeBytecode: Optional[Bytecode] = None
     # abi, userdoc and devdoc must conform to spec
@@ -76,6 +80,8 @@ def from_dict(cls, params: Dict):
         params = deepcopy(params)
         update_params(params, "deploymentBytecode", Bytecode)
         update_params(params, "runtimeBytecode", Bytecode)
+        if params.get("sourcePath"):
+            params["sourcePath"] = Path(params["sourcePath"])
         return cls(**params)  # type: ignore
 
 
@@ -103,6 +109,22 @@ def load_content(self):
         response = urllib.request.urlopen(self.urls[0])
         self.content = response.read().decode("utf-8")
 
+    def compute_checksum(self, algorithm: str = "md5", force: bool = False):
+        """
+        Compute the checksum if `content` exists but `checksum` doesn't
+        exist yet. Or compute the checksum regardless if `force` is `True`.
+        """
+        if self.checksum and not force:
+            return  # skip recalculating
+
+        if not self.content:
+            raise ValueError("Content not loaded yet. Can't compute checksum.")
+
+        self.checksum = Checksum(  # type: ignore
+            hash=compute_checksum(self.content.encode("utf8"), algorithm=algorithm),
+            algorithm=algorithm,
+        )
+
     @classmethod
     def from_dict(cls, params: Dict):
         params = deepcopy(params)

src/ape/types/manifest.py

@@ -47,13 +47,28 @@ class PackageManifest(FileMixin, SerializableType):
     #       version as `manifest`
     buildDependencies: Optional[Dict[str, str]] = None
 
+    def __getattr__(self, attr_name: str):
+        if self.contractTypes and attr_name in self.contractTypes:
+            return self.contractTypes[attr_name]
+
+        else:
+            raise AttributeError(f"{self.__class__.__name__} has no attribute '{attr_name}'")
+
     def to_dict(self):
         data = super().to_dict()
 
         if "contractTypes" in data and data["contractTypes"]:
             for name in data["contractTypes"]:
                 # NOTE: This was inserted by us, remove it
                 del data["contractTypes"][name]["contractName"]
+                # convert Path to str, or else we can't serialize this as JSON
+                data["contractTypes"][name]["sourceId"] = str(
+                    data["contractTypes"][name]["sourceId"]
+                )
+                if "sourcePath" in data["contractTypes"][name]:
+                    data["contractTypes"][name]["sourcePath"] = str(
+                        data["contractTypes"][name]["sourcePath"]
+                    )
 
         return data