Skip to content

Commit 0665cd3

Browse files
john-sandalljohn-sandall
authored
Merge pull request #40 from HumbleData/upgrade-python--and-deps
Various upgrades (Python version, dependencies, Poetry, pre-commit, and more)
2 parents be727a7 + c1e21e0 commit 0665cd3

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

67 files changed

+4028
-550
lines changed

‎.pre-commit-config.yaml‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
# Update all versions in this file by running:
2+
# $ pre-commit autoupdate
3+
minimum_pre_commit_version: 2.19.0
4+
default_language_version:
5+
python: python3.9
6+
repos:
7+
- repo: https://github.com/pre-commit/pre-commit-hooks
8+
rev: v4.2.0
9+
hooks:
10+
- id: end-of-file-fixer
11+
name: Check for a blank line at the end of scripts (auto-fixes)
12+
exclude: ^solutions/
13+
- id: trailing-whitespace
14+
- id: check-builtin-literals
15+
exclude: |
16+
(?x)^(
17+
solutions/01_04.py|
18+
solutions/02_21.py|
19+
solutions/04_05.py|
20+
solutions/05_01.py|
21+
solutions/05_12.py
22+
)$
23+
- id: check-byte-order-marker
24+
- id: check-case-conflict
25+
- id: check-merge-conflict
26+
- id: check-symlinks
27+
- id: check-toml
28+
- id: check-vcs-permalinks
29+
- id: check-xml
30+
- id: debug-statements
31+
exclude: |
32+
(?x)^(
33+
solutions/01_04.py|
34+
solutions/02_21.py|
35+
solutions/04_05.py|
36+
solutions/05_01.py|
37+
solutions/05_12.py
38+
)$
39+
- id: detect-private-key
40+
- id: mixed-line-ending
41+
- id: fix-encoding-pragma
42+
args: ["--remove"]
43+
- id: check-yaml
44+
- id: check-added-large-files
45+
name: Check for files larger than 5 MB
46+
args: ["--maxkb=5120"]
47+
- id: check-ast
48+
exclude: |
49+
(?x)^(
50+
solutions/01_04.py|
51+
solutions/02_21.py|
52+
solutions/04_05.py|
53+
solutions/05_01.py|
54+
solutions/05_12.py
55+
)$
56+
- id: check-docstring-first
57+
- repo: https://github.com/myint/autoflake
58+
rev: v1.4
59+
hooks:
60+
- id: autoflake
61+
args: &autoflake
62+
- --in-place
63+
- --remove-all-unused-imports
64+
- --expand-star-imports
65+
- --remove-duplicate-keys
66+
- --remove-unused-variables
67+
exclude: |
68+
(?x)^(
69+
solutions/02_01.py
70+
)$
71+
- repo: https://gitlab.com/pycqa/flake8
72+
rev: 3.9.2
73+
hooks:
74+
- &flake8
75+
id: flake8
76+
additional_dependencies:
77+
- flake8-docstrings==1.6.0
78+
- flake8-eradicate==1.2.0
79+
- flake8-fixme==1.1.1
80+
- flake8-implicit-str-concat==0.2.0
81+
- flake8-print==4.0.0
82+
- flake8-return==1.1.3
83+
args: ["--config=setup.cfg"]
84+
- repo: local
85+
hooks:
86+
- id: linestripper-add
87+
name: linestripper-add
88+
entry: python linestripper.py add
89+
language: system
90+
always_run: true
91+
pass_filenames: false
92+
- repo: https://github.com/psf/black
93+
rev: 22.3.0
94+
hooks:
95+
- id: black
96+
exclude: |
97+
(?x)^(
98+
solutions/01_04.py|
99+
solutions/01_05.py|
100+
solutions/02_21.py|
101+
solutions/04_05.py|
102+
solutions/05_01.py|
103+
solutions/05_12.py
104+
)$
105+
- repo: local
106+
hooks:
107+
- id: linestripper-remove
108+
name: linestripper-remove
109+
entry: python linestripper.py remove
110+
language: system
111+
always_run: true
112+
pass_filenames: false
113+
- repo: https://github.com/asottile/add-trailing-comma
114+
rev: v2.2.3
115+
hooks:
116+
- id: add-trailing-comma
117+
args: [--py36-plus]
118+
- repo: https://github.com/timothycrosley/isort
119+
rev: 5.10.1
120+
hooks:
121+
- id: isort
122+
types: [python]
123+
- repo: https://github.com/asottile/pyupgrade
124+
rev: v2.32.1
125+
hooks:
126+
- id: pyupgrade
127+
args:
128+
- "--py39-plus"
129+
- repo: https://github.com/Cielquan/mirrors-poetry
130+
rev: 1.1.13
131+
hooks:
132+
- id: poetry-check
133+
- id: poetry-lock
134+
args: ["--no-update"]
135+
- id: poetry-export
136+
args: ["--without-hashes"]
137+
- repo: https://github.com/pre-commit/mirrors-prettier
138+
rev: v2.6.2
139+
hooks:
140+
- id: prettier
141+
types_or: [yaml]
142+
additional_dependencies:
143+
- "[email protected]"
144+
- repo: local
145+
hooks:
146+
- id: pylint
147+
name: pylint
148+
entry: pylint solutions
149+
language: system
150+
always_run: true
151+
pass_filenames: false
152+
# stages: [manual]
153+
- id: pip-audit
154+
name: pip-audit
155+
entry: pip-audit
156+
language: system
157+
always_run: true
158+
pass_filenames: false

‎.python-version‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
3.8.5
1+
3.9.13

‎.vscode/settings.json‎

Lines changed: 44 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,49 @@
11
{
2-
"python.linting.enabled": true,
3-
"python.linting.flake8Enabled": true,
4-
"python.linting.flake8Args": [],
5-
"python.linting.pylintArgs": [],
6-
"python.linting.pylintUseMinimalCheckers": false,
7-
"python.formatting.provider": "black",
8-
"python.formatting.blackArgs": [],
9-
"editor.formatOnSave": true,
10-
"python.sortImports.args": [
11-
"--recursive",
12-
],
13-
"[python]": {
14-
"editor.codeActionsOnSave": {
15-
"source.organizeImports": true
2+
// editor & autoformatter settings
3+
"editor.formatOnSave": true,
4+
"editor.trimAutoWhitespace": true,
5+
"files.trimTrailingWhitespace": true,
6+
"prettier.enable": true,
7+
8+
// python - black
9+
"python.formatting.provider": "black",
10+
"python.formatting.blackArgs": [],
11+
12+
// python - other
13+
"python.languageServer": "Pylance",
14+
15+
// python - linting & static analysis
16+
"python.analysis.extraPaths": ["solutions"],
17+
"python.analysis.typeCheckingMode": "basic",
18+
"python.linting.enabled": true,
19+
"python.linting.flake8Enabled": true,
20+
"python.linting.flake8Args": [],
21+
"python.linting.pylintEnabled": true,
22+
"python.linting.pylintArgs": [],
23+
24+
// Python files only
25+
"[python]": {
26+
// isort on save
27+
"editor.codeActionsOnSave": {"source.organizeImports": true},
28+
// Stop the 'Black does not support "Format Selection"' message every
29+
// time you paste something (https://stackoverflow.com/a/63612401/3279076)
30+
"editor.formatOnPaste": false
31+
},
32+
33+
// python - pytest (https://code.visualstudio.com/docs/python/testing)
34+
"python.testing.unittestEnabled": false,
35+
"python.testing.pytestEnabled": true,
36+
"python.testing.pytestArgs": ["tests"],
37+
38+
// git
39+
"git.enabled": true,
40+
41+
// file associations
42+
"files.associations": {
43+
"**/requirements{/**,*}.{txt,in}": "pip-requirements",
1644
},
17-
// Stop the 'Black does not support "Format Selection"' message every
18-
// time you paste something (https://stackoverflow.com/a/63612401/3279076)
19-
"editor.formatOnPaste": false
20-
},
21-
"python.pythonPath": "${env:VIRTUAL_ENV}",
22-
// from here: https://github.com/microsoft/vscode-python/wiki/AB-Experiments
23-
// being selected made things run differently on different peoples systems.
24-
"python.experiments.enabled": false,
45+
46+
"cSpell.language": "en-GB",
2547
"cSpell.words": [
2648
"adelie",
2749
"boxplot",

‎CHANGELOG.md‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# Changelog
2+
3+
All notable changes to this project will be documented in this file.
4+
5+
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
6+
and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
7+
8+
<!-- TOWNCRIER -->
9+
10+
## [1.1.0] - 2022-07-10
11+
12+
- Various upgrades ([#40](https://github.com/HumbleData/beginners-data-workshop/pull/40))
13+
- **Behind-the-scenes**
14+
- Upgrade Python to 3.9.13
15+
- Upgrade all dependencies to latest with NumPy/pandas/Matplotlib/scikit-learn matching CoCalc versions.
16+
- Switch from `pip-tools` to Poetry (configuration in `pyproject.toml`)
17+
- Update Development Setup guidance in `README.md`
18+
- Integrate & configure flake8, pylint, black & other linters.
19+
- Added pre-commit configuration.
20+
- Updated VS Code `settings.json`
21+
- Added `linestripper.py` to keep EOF newlines out of solutions code (better attendee UX) yet black-compliant.
22+
- Added `CHANGELOG.md` and started versioning releases.
23+
- **Workshop materials**
24+
- Reviewed all workshop materials for deprecations.
25+
- Replaced all single quotes in solutions with double quotes for black compliance.
26+
- Added EOF newlines to datasets.
27+
- Other minor changes: trailing whitespace, trailing commas, isort compliant solution code.
28+
29+
30+
## [1.0.0] - 2021-07-23
31+
Final version used for Humble Data workshops in 2021.

‎README.md‎

Lines changed: 31 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -25,36 +25,39 @@ pip-compile
2525
pip-sync
2626
```
2727

28-
## Setup
28+
## Attendee Setup
29+
30+
1. Visit [https://bit.ly/humble-python-setup] to install Python onto your machine.
31+
2. The notebooks in this project should work great with Anaconda out of the box. If you want to create your own virtual
32+
environment for this workshop, your mentor will guide you through the simplest setup.
33+
34+
> Note: The Development Setup outlined below will work for you, but is more complex than what you need to follow this
35+
workshop. This setup is designed to make it easy to upgrade Python dependencies (e.g. pandas, Seaborn), and run
36+
automated checks to highlight if any code in the notebooks is broken (we call this a "test suite").
37+
38+
## Development Setup
39+
40+
This project uses [pyenv](https://github.com/pyenv/pyenv), [pyenv-virtualenvwrapper](https://github.com/pyenv/pyenv-virtualenvwrapper)
41+
and [Poetry](https://python-poetry.org/docs/). Please [see here](https://github.com/CoefficientSystems/coefficient-cookiecutter/blob/develop/%7B%7Bcookiecutter.repo_name%7D%7D/docs/getting_started.md) for a step-by-step installation guide.
2942

3043
```bash
31-
# Install virtualenv
32-
pip install virtualenv
33-
34-
35-
# Install virtualenvwrapper (http://virtualenvwrapper.readthedocs.org/en/latest/index.html)
36-
pip install virtualenvwrapper
37-
# Tell shell to source virtualenvwrapper.sh and where to put the virtualenvs by adding following to .zshrc
38-
zshconfig
39-
# # "Tell shell to source virtualenvwrapper.sh and where to put the virtualenvs"
40-
# export WORKON_HOME=$HOME/.virtualenvs
41-
# export PROJECT_HOME=$HOME/code
42-
# source /usr/local/bin/virtualenvwrapper.sh
43-
source ~/.zshrc
44-
source /usr/local/bin/virtualenvwrapper.sh
45-
# Now let's make a virtualenv
46-
mkvirtualenv venv
47-
workon venv
48-
# Commands `workon venv`, `deactivate`, `lsvirtualenv` and `rmvirtualenv` are useful
49-
# WARNING: When you brew install formulae that provide Python bindings, you should not be in an active virtual environment.
50-
# (https://github.com/Homebrew/homebrew/blob/master/share/doc/homebrew/Homebrew-and-Python.md)
51-
deactivate
52-
53-
54-
# Create virtualenv & install packages
55-
mkvirtualenv beginners-data-workshop
56-
pip install pip-tools
57-
pip-sync
44+
# Install Python
45+
pyenv install $(cat .python-version)
46+
pyenv shell $(cat .python-version)
47+
python -m pip install --upgrade pip
48+
python -m pip install virtualenvwrapper
49+
pyenv virtualenvwrapper
50+
51+
# Setup the virtualenv
52+
mkvirtualenv -p python$(cat .python-version) $(cat .venv)
53+
python -V
54+
python -m pip install --upgrade pip
55+
56+
# Install dependencies with Poetry
57+
poetry self update
58+
poetry install --no-root --remove-untracked
59+
60+
# Create Jupyter Kernel
5861
python -m ipykernel install --user --name beginners-data-workshop --display-name "Python (beginners-data-workshop)"
5962

6063
# Install Jupyter Extensions static files

‎data/food_training/languages.csv‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,4 +47,4 @@ Switzerland,German/French/Italian/Romansch
4747
Turkey,Turkish
4848
Ukraine,Ukrainian
4949
United Kingdom,English
50-
Vatican City State,Latin/Italian
50+
Vatican City State,Latin/Italian

‎data/food_training/training_2014.csv‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,4 +27,4 @@ RASFF,Madrid,2015-03-23,2015-03-26,1,,
2727
Contingency Planning,Padua,2015-03-23,2015-03-27,1,,
2828
Microbiological Criteria,Lisbon,2015-03-23,2015-03-27,1,,
2929
Food Composition and Information,Athens,2015-03-23,2015-03-27,1,,
30-
Control on Contaminants in Feed and Food,Berlin,2015-03-24,2015-03-27,3,,
30+
Control on Contaminants in Feed and Food,Berlin,2015-03-24,2015-03-27,3,,

‎data/food_training/training_2015.csv‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,4 +121,4 @@ Plant Health Controls,Venice/Treviso,2016-03-07,2016-03-11,2,,
121121
Import Controls on Food and Feed of Non-Animal Origin,Genoa;Italy,2016-03-08,2016-03-11,4,,
122122
Feed Law,Nantes;France,2016-03-14,2016-03-18,3,,
123123
Animal by Products (Upgraded),Maribor;Slovenia,2016-03-15,2016-03-18,2,,
124-
EU Feed Hygiene Rules and HACCP auditing ,Barcelona,2016-03-31,2016-06-03,2,,
124+
EU Feed Hygiene Rules and HACCP auditing ,Barcelona,2016-03-31,2016-06-03,2,,

‎data/food_training/training_2016.csv‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,4 +115,4 @@ Contingency Planning,Venice;IT,2017-03-06,2017-03-10,3,,
115115
Audit B1 - Standard Level,Valencia,2017-03-06,2017-03-10,2,,
116116
Audit A,Bratislava,2017-03-20,2017-03-24,1,,
117117
Food Hygiene and Flexibility,Zagreb/Helsinki,2017-03-20,2017-03-24,2,,
118-
Animal Health and Disease Prevention for Bees and Zoo Animals,Antwerp,2017-03-28,2017-03-31,2,,
118+
Animal Health and Disease Prevention for Bees and Zoo Animals,Antwerp,2017-03-28,2017-03-31,2,,

0 commit comments

Comments
 (0)