apply pre-commit

tchaton · tchaton · commit 1579e972bc58 · 2020-10-08T18:37:55.000+01:00
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -28,5 +28,9 @@ RUN python3 -m pip install -U pip \
 
 WORKDIR /app
 
-COPY requirements.txt .
-RUN $PYTHON -m pip install -r requirements.txt && rm -rf /root/.cache
+COPY pyproject.toml .
+COPY poetry.lock .
+COPY scripts/clean_requirements.sh .
+RUN $PYTHON -m poetry export -f requirements.txt -o requirements.txt --without-hashes
+RUN sh clean_requirements.sh
+RUN $PYTHON -m pip install -r requirements.txt && rm -rf /root/.cache
diff --git a/.dockerignore b/.dockerignore
@@ -247,4 +247,4 @@ model.tar.gz
 data
 checkpoints
 .pytest_cache
-pytestdebug.log
+pytestdebug.log
diff --git a/.gitignore b/.gitignore
@@ -38,4 +38,4 @@ checkpoints
 .pytest_cache
 pytestdebug.log
 *log
-.venv
+.venv
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,51 @@
+exclude: 'benchmark|conf|data|docs|outputs'
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v1.2.3
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-added-large-files
+        args: [--maxkb=15000]
+      - id: check-yaml
+
+  - repo: https://github.com/ambv/black
+    rev: stable
+    hooks:
+      - id: black
+        language_version: python3.7
+
+  - repo: https://github.com/humitos/mirrors-autoflake.git
+    rev: v1.3
+    hooks:
+      - id: autoflake
+        args:
+          [
+            '--in-place',
+            '--remove-unused-variable',
+            '--ignore-init-module-imports',
+            '--imports=torch,torch_geometric,torch_scatter,torch_cluster,numpy,sklearn,scipy,torch_sparse,torch_points_kernels',
+          ]
+  - repo: https://github.com/kynan/nbstripout
+    rev: master
+    hooks:
+      - id: nbstripout
+        files: '.ipynb'
+
+  - repo: local
+    hooks:
+      - id: requirements.txt
+        name: Generate requirements.txt
+        entry: poetry export
+        args:
+          [
+            '-f',
+            'requirements.txt',
+            '-o',
+            'requirements.txt',
+            '--without-hashes',
+          ]
+        pass_filenames: false
+        language: system
+        files: 'poetry.lock'
diff --git a/Makefile b/Makefile
@@ -11,4 +11,4 @@ workflow-test-xdist:
 	coverage run -m pytest -n `python -c 'import multiprocessing as mp; print(mp.cpu_count())'` --color=yes
 
 workflow-test:
-	coverage run -m pytest --color=yes
+	coverage run -m pytest --color=yes
diff --git a/pyproject.toml b/pyproject.toml
@@ -23,4 +23,4 @@ build-backend = "poetry.masonry.api"
 
 [[tool.poetry.source]]
 name = 'private'
-url = 'https://pypi.org/project'
+url = 'https://pypi.org/project'
diff --git a/requirements.txt b/requirements.txt
@@ -0,0 +1,72 @@
+absl-py==0.10.0
+antlr4-python3-runtime==4.8
+apipkg==1.5
+atomicwrites==1.4.0; sys_platform == "win32"
+attrs==20.2.0
+cachetools==4.1.1
+certifi==2020.6.20
+chardet==3.0.4
+click==7.1.2
+colorama==0.4.3; sys_platform == "win32"
+configparser==5.0.0
+coverage==5.3
+docker-pycreds==0.4.0
+execnet==1.7.1
+fsspec==0.8.3
+future==0.18.2
+gitdb==4.0.5
+gitpython==3.1.9
+google-auth-oauthlib==0.4.1
+google-auth==1.22.1
+grpcio==1.32.0
+hydra-core==1.0.3
+idna==2.10
+importlib-metadata==2.0.0; python_version < "3.8"
+importlib-resources==3.0.0; python_version < "3.9"
+iniconfig==1.0.1
+joblib==0.17.0
+markdown==3.3
+numpy==1.19.2
+oauthlib==3.1.0
+omegaconf==2.0.2
+packaging==20.4
+pathtools==0.1.2
+pillow==7.2.0
+pluggy==0.13.1
+promise==2.3
+protobuf==3.13.0
+psutil==5.7.2
+py==1.9.0
+pyasn1-modules==0.2.8
+pyasn1==0.4.8
+pyparsing==2.4.7
+pytest-forked==1.3.0
+pytest-xdist==2.1.0
+pytest==6.1.1
+python-dateutil==2.8.1
+pytorch-lightning==0.10.0
+pyyaml==5.3.1
+requests-oauthlib==1.3.0
+requests==2.24.0
+rsa==4.6; python_version >= "3.5"
+scikit-learn==0.23.2
+scipy==1.5.2
+sentry-sdk==0.18.0
+shortuuid==1.0.1
+six==1.15.0
+smmap==3.0.4
+subprocess32==3.5.4
+tensorboard-plugin-wit==1.7.0
+tensorboard==2.2.0
+threadpoolctl==2.1.0
+toml==0.10.1
+torch==1.6.0
+torchvision==0.7.0
+tqdm==4.50.1
+typing-extensions==3.7.4.3
+urllib3==1.25.10
+wandb==0.10.5
+watchdog==0.10.3
+werkzeug==1.0.1
+zipp==3.3.0
+zipp==3.3.0; python_version < "3.8"
diff --git a/scripts/clean_requirements.sh b/scripts/clean_requirements.sh
@@ -1,2 +1,2 @@
-python -c "print(''.join([l.split(';')[0]+'\n' if ('python_version' in l or 'sys_platform' in l) else l for l in open('requirements.txt', 'r').readlines()]))" > requirements_tmp.txt		
-mv requirements_tmp.txt requirements.txt
+python -c "print(''.join([l.split(';')[0]+'\n' if ('python_version' in l or 'sys_platform' in l) else l for l in open('requirements.txt', 'r').readlines()]))" > requirements_tmp.txt
+mv requirements_tmp.txt requirements.txt
diff --git a/src/__init__.py b/src/__init__.py
@@ -7,13 +7,15 @@
 from src.core.base_dataset import BaseDataset
 from src.core.base_model import BaseModel
 
+
 def custom_instantiate(dataset_cfg, task_cfg):
     dataset_dict = OmegaConf.to_container(dataset_cfg, resolve=True)
     task_dict = OmegaConf.to_container(task_cfg, resolve=True)
     dataset_dict["task_config"] = task_dict
     t_cls = get_class(dataset_dict["_target_"])
     return t_cls(**dataset_dict)
 
+
 def attach_step_and_epoch_functions(model, datamodule):
     datamodule.forward = model.forward
     datamodule.log = model.log
@@ -22,19 +24,25 @@ def attach_step_and_epoch_functions(model, datamodule):
             if not is_overridden(attr, model):
                 setattr(model, attr, getattr(datamodule, attr))
 
+
 def check_task_possible(task, cfg):
-    assert task in [c._target_ for c in cfg], f"Provided {task} isn't supported by {cfg}"
+    assert task in [
+        c._target_ for c in cfg
+    ], f"Provided {task} isn't supported by {cfg}"
+
 
 def initialize_task(cfg: DictConfig):
     # Extract task mixin
-    check_task_possible(cfg.task._target_, cfg.model.authorized_tasks) 
-    check_task_possible(cfg.task._target_, cfg.dataset.authorized_tasks) 
+    check_task_possible(cfg.task._target_, cfg.model.authorized_tasks)
+    check_task_possible(cfg.task._target_, cfg.dataset.authorized_tasks)
 
     data_module: BaseDataset = custom_instantiate(cfg.dataset, cfg.task)
-    model: BaseModel = instantiate(cfg.model, **data_module.hyper_parameters, optimizers=cfg.optimizers.optimizers)
+    model: BaseModel = instantiate(
+        cfg.model, **data_module.hyper_parameters, optimizers=cfg.optimizers.optimizers
+    )
 
     attach_step_and_epoch_functions(model, data_module)
 
     data_module.prepare_data()
 
-    return model, data_module
+    return model, data_module
diff --git a/src/core/base_model.py b/src/core/base_model.py
@@ -1,14 +1,9 @@
-import os.path as osp
 from omegaconf import OmegaConf, DictConfig, ListConfig
 from functools import partial
 from hydra.utils import instantiate, get_class
-import torch
-from torch import nn
-from torch.nn import Module
-from torch.nn import Sequential, ModuleDict, ModuleList
-import torch.nn.functional as F
 from pytorch_lightning import LightningModule
 
+
 def delete_key(d, key):
     try:
         del d[key]
@@ -28,8 +23,7 @@ def __init__(self, *args, **kwargs):
         if len(self._optimizers_conf) == 1:
             self._optimizer_name = self._optimizers_conf[0]["name"]
         self._init_optim = partial(
-            self._init_optim,
-            optimizers_conf=self._optimizers_conf,
+            self._init_optim, optimizers_conf=self._optimizers_conf,
         )
 
     @property
@@ -117,4 +111,3 @@ def _sanetize_kwargs(value):
         for key, value in kwargs.items():
             kwargs[key] = _sanetize_kwargs(value)
         return kwargs
-
diff --git a/src/core/base_task.py b/src/core/base_task.py
@@ -1,5 +1,6 @@
 from hydra.utils import get_class
 
+
 class BaseTasksMixin:
     def __init__(self, *args, **kwargs):
 
@@ -9,4 +10,4 @@ def __init__(self, *args, **kwargs):
         if len(self.__class__.__bases__) > 1:
             self.__class__.__bases__ = (self.__class__.__bases__[0],)
         self.__class__.__bases__ += (task_target_cls,)
-        task_target_cls.__init__(self, **task_config)
+        task_target_cls.__init__(self, **task_config)
diff --git a/src/core/transforms.py b/src/core/transforms.py
@@ -1,6 +1,7 @@
 from typing import List, Optional
 import torch
 
+
 class AddFeatsByKeys(object):
     """This transform takes a list of attributes names and if allowed, add them to x
     Example:
@@ -59,8 +60,12 @@ def __init__(
             stricts = [True for _ in range(num_names)]
 
         transforms = [
-            AddFeatByKey(add_to_x, feat_name, input_nc_feat=input_nc_feat, strict=strict)
-            for add_to_x, feat_name, input_nc_feat, strict in zip(list_add_to_x, feat_names, input_nc_feats, stricts)
+            AddFeatByKey(
+                add_to_x, feat_name, input_nc_feat=input_nc_feat, strict=strict
+            )
+            for add_to_x, feat_name, input_nc_feat, strict in zip(
+                list_add_to_x, feat_names, input_nc_feats, stricts
+            )
         ]
 
         self.transform = Compose(transforms)
@@ -107,14 +112,20 @@ def __call__(self, data: Data):
         feat = getattr(data, self._feat_name, None)
         if feat is None:
             if self._strict:
-                raise Exception("Data should contain the attribute {}".format(self._feat_name))
+                raise Exception(
+                    "Data should contain the attribute {}".format(self._feat_name)
+                )
             else:
                 return data
         else:
             if self._input_nc_feat:
                 feat_dim = 1 if feat.dim() == 1 else feat.shape[-1]
                 if self._input_nc_feat != feat_dim and self._strict:
-                    raise Exception("The shape of feat: {} doesn t match {}".format(feat.shape, self._input_nc_feat))
+                    raise Exception(
+                        "The shape of feat: {} doesn t match {}".format(
+                            feat.shape, self._input_nc_feat
+                        )
+                    )
             x = getattr(data, "x", None)
             if x is None:
                 if self._strict and data.pos.shape[0] != feat.shape[0]:
@@ -142,7 +153,8 @@ def __repr__(self):
             self.__class__.__name__, self._add_to_x, self._feat_name, self._strict
         )
 
+
 def normalized_cut_2d(edge_index, pos):
     row, col = edge_index
     edge_attr = torch.norm(pos[row] - pos[col], p=2, dim=1)
-    return normalized_cut(edge_index, edge_attr, num_nodes=pos.size(0))
+    return normalized_cut(edge_index, edge_attr, num_nodes=pos.size(0))
diff --git a/src/models/simple_mlp.py b/src/models/simple_mlp.py
@@ -1,7 +1,4 @@
-import numpy as np
-import torch
 from torch import nn
-from torch.nn import functional as F
 from src.core.base_model import BaseModel
 
 
@@ -20,7 +17,9 @@ def MLP(channels, activation=nn.LeakyReLU(0.2), bn_momentum=0.1, bias=True):
 
 
 class SimpleMLP(BaseModel):
-    def __init__(self, *args, num_features: int = None, num_classes: int = None, **kwargs):
+    def __init__(
+        self, *args, num_features: int = None, num_classes: int = None, **kwargs
+    ):
         super(SimpleMLP, self).__init__(*args, **kwargs)
 
         self.save_hyperparameters()
diff --git a/src/tasks/__init__.py b/src/tasks/__init__.py
@@ -1,2 +1,2 @@
 from .bases import *
-from .classification import CategoricalClassificationStepsMixin
+from .classification import CategoricalClassificationStepsMixin
diff --git a/src/tasks/bases.py b/src/tasks/bases.py
@@ -1 +0,0 @@
-
diff --git a/src/tasks/classification.py b/src/tasks/classification.py
@@ -1,13 +1,7 @@
-import os
-import os.path as osp
-import numpy as np
-from functools import partial
-import numpy as np
-import torch
 import torch.nn.functional as F
 from pytorch_lightning import metrics
 import pytorch_lightning as pl
-from collections import namedtuple
+
 
 class CategoricalClassificationStepsMixin:
     def __init__(self, *args, **kwargs):
@@ -28,11 +22,10 @@ def validation_step(self, batch, batch_idx):
         x, y = batch
         y_hat = self(x)
         loss = F.cross_entropy(y_hat, y)
-        self.log('valid_loss', loss.item(), self._prog_bar)
+        self.log("valid_loss", loss.item(), self._prog_bar)
 
     def test_step(self, batch, batch_idx):
         x, y = batch
         y_hat = self(x)
         loss = F.cross_entropy(y_hat, y)
-        self.log('test_loss', loss, self._prog_bar)
-
+        self.log("test_loss", loss, self._prog_bar)
diff --git a/src/utils/jit.py b/src/utils/jit.py
@@ -1,4 +1,3 @@
-
 import os
 import torch
 import logging
@@ -13,9 +12,10 @@ def check_jittable(model, data_module):
     data_module.forward = model.forward
     batch = get_single_batch(data_module)
     _ = data_module.test_step(batch, None)
-    
+
     log.info("Check model is jittable complete.")
 
+
 def get_single_batch(data_module):
     for batch in data_module.test_dataloader():
         return batch
diff --git a/src/utils/loggers.py b/src/utils/loggers.py
diff --git a/test/test_inference.py b/test/test_inference.py
diff --git a/train.py b/train.py
