Skip to content

Commit ba78da0

Browse files
Expl0dingBananaExpl0dingBanana
authoredDec 17, 2020
Workflow improvements (#1066)
* Added Makefile to easily run tests and improve code quality * Fix build for MAD docker image * Update development build to use the MAD docker image
1 parent 293df9c commit ba78da0

22 files changed

+494
-60
lines changed
 

‎.flake8

-4
This file was deleted.

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ configs/coords/*
2323
status.json
2424
update_log.json
2525
commands.json
26+
.tox/
2627
# Byte-compiled / optimized / DLL files
2728
__pycache__/
2829
*.py[cod]

‎.gitlint

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
[general]
2+
ignore=title-trailing-punctuation, title-must-not-contain-word
3+
4+
[title-max-length]
5+
line-length=120
6+
7+
[body-max-line-length]
8+
line-length=140

‎.gitmessage

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# (ISSUE IF APPLICABLE) SHORT_DESCRIPTION
2+
3+
# LONG_DESCRIPTION
4+
# * Thing 1
5+
# * Thing 2

‎.isort.cfg

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
[settings]
2+
known_third_party = PIL,aioconsole,apkutils,bitstring,configargparse,cv2,dataclasses,flask,flask_caching,gevent,google,gpapi,gpxdata,imutils,loguru,mysql,numpy,pkg_resources,psutil,pytesseract,pytest,requests,s2sphere,urllib3,websockets,werkzeug

‎.pre-commit-config.yaml

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
repos:
2+
- repo: https://github.com/Seeefo/manage-commit-msg.git
3+
rev: 1.0.3
4+
hooks:
5+
- id: restore-commit-msg
6+
- repo: https://github.com/jorisroovers/gitlint
7+
rev: v0.13.1
8+
hooks:
9+
- id: gitlint
10+
- repo: https://github.com/Seeefo/manage-commit-msg.git
11+
rev: 1.0.3
12+
hooks:
13+
- id: save-commit-msg
14+
- repo: https://gitlab.com/pycqa/flake8
15+
rev: 3.7.9
16+
hooks:
17+
- id: flake8
18+
language_version: python3
19+
additional_dependencies: [flake8-bugbear, flake8-logging-format, flake8-variables-names, pep8-naming, flake8-eradicate]
20+
- repo: https://github.com/asottile/seed-isort-config
21+
rev: v2.1.0
22+
hooks:
23+
- id: seed-isort-config
24+
language_version: python3
25+
- repo: https://github.com/pre-commit/mirrors-isort
26+
rev: v4.3.21
27+
hooks:
28+
- id: isort
29+
language_version: python3
30+
- repo: https://github.com/pre-commit/pre-commit-hooks
31+
rev: v2.5.0
32+
hooks:
33+
- id: trailing-whitespace
34+
- id: end-of-file-fixer

‎Makefile

+149
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
CMD ?= bash
2+
CONTAINER_NAME ?= mapadroid-dev
3+
include docker/.dev.env
4+
export
5+
6+
define PIP_MISSING
7+
Pip is missing or not available in PATH. If pip is not installed
8+
instructions can be found at https://pip.pypa.io/en/stable/installing/
9+
endef
10+
11+
define PRE_COMMIT_MISSING
12+
Install pre-commit @ https://pre-commit.com/#install then run
13+
source ~/.profile
14+
endef
15+
16+
define DOCKER_MISSING
17+
Docker installation is missing or not available. This could be caused by
18+
not having docker installed or the user does not have access to docker.
19+
Installation instructions can be found at
20+
https://docs.docker.com/get-docker/
21+
endef
22+
23+
define DOCKER_NOT_RUNNING
24+
Docker is not running or the user does not have access to the Docker
25+
Engine. Please verify that its running and you have access. On *nix
26+
systems you can run the following commands to grant access:
27+
sudo groupadd docker
28+
sudo usermod -aG docker $USER
29+
newgrp docker
30+
endef
31+
32+
define DOCKER_COMPOSE_MISSING
33+
docker-compose -f ${COMPOSE_FILE_DEV} is not installed or not available in PATH. Installation
34+
instructions can be found at https://docs.docker.com/compose/install/
35+
If docker-compose -f ${COMPOSE_FILE_DEV} is installed PATH needs to be corrected to include
36+
the binary
37+
endef
38+
39+
define DOCKER_COMPOSE_OLD
40+
docker-compose -f ${COMPOSE_FILE_DEV} is too old. Update docker-compose -f ${COMPOSE_FILE_DEV} from the instructions at
41+
https://docs.docker.com/compose/install/
42+
endef
43+
44+
# Windows defines OS but *nix does not
45+
ifdef OS
46+
SHELL := powershell.exe
47+
pip := $(shell Get-Command pip | Select-Object -ExpandProperty Source)
48+
precommit := $(shell Get-Command pre-commit | Select-Object -ExpandProperty Source)
49+
docker := $(shell Get-Command docker | Select-Object -ExpandProperty Source)
50+
docker_compose := $(shell Get-Command docker-compose -f ${COMPOSE_FILE_DEV} | Select-Object -ExpandProperty Source)
51+
UID ?= 1000
52+
GID ?= 1000
53+
else
54+
ifneq ($(shell which pip),)
55+
pip := $(shell which pip)
56+
else ifneq ($(shell which pip3),)
57+
pip := $(shell which pip3)
58+
else
59+
$(error $(PIP_MISSING))
60+
endif
61+
precommit := $(shell which pre-commit)
62+
docker := $(shell which docker)
63+
docker_compose := $(shell which docker-compose -f ${COMPOSE_FILE_DEV})
64+
UID ?= $(shell id -u)
65+
GID ?= $(shell id -g)
66+
endif
67+
68+
69+
ifeq (, $(pip))
70+
$(error $(PIP_MISSING))
71+
endif
72+
ifneq ($(VIRTUAL_ENV), )
73+
pip_precommit_installation := $(pip) install pre-commit
74+
else
75+
pip_precommit_installation := $(pip) install --user pre-commit
76+
endif
77+
ifeq (, $(precommit))
78+
$(error $(PRE_COMMIT_MISSING))
79+
endif
80+
ifeq (, $(docker))
81+
$(error $(DOCKER_MISSING))
82+
endif
83+
ifeq (, $(shell docker info))
84+
$(error $(DOCKER_NOT_RUNNING))
85+
endif
86+
ifeq (, $(docker_compose))
87+
$(error $(DOCKER_COMPOSE_MISSING))
88+
endif
89+
compose_ver ?= $(shell docker-compose -f ${COMPOSE_FILE_DEV} --version | cut -d' ' -f3 | cut -d '.' -f2)
90+
ifeq (, compose_ver < 27)
91+
$(error $(DOCKER_COMPOSE_OLD))
92+
endif
93+
94+
clean: clean-tox down
95+
96+
clean-tox:
97+
rm -rf .tox
98+
99+
build:
100+
docker build --file docker/Dockerfile --tag ${LOCAL_MAD_IMAGE} .
101+
docker-compose -f ${COMPOSE_FILE_DEV} build --no-cache
102+
103+
rebuild:
104+
docker build --file docker/Dockerfile --tag ${LOCAL_MAD_IMAGE} .
105+
docker-compose -f ${COMPOSE_FILE_DEV} build
106+
107+
setup-precommit:
108+
$(pip_precommit_installation)
109+
pre-commit install
110+
pre-commit install --hook-type commit-msg
111+
112+
setup: setup-precommit
113+
git config commit.template .gitmessage
114+
115+
up:
116+
docker-compose -f ${COMPOSE_FILE_DEV} up --detach
117+
118+
shell: up
119+
docker-compose -f ${COMPOSE_FILE_DEV} exec -u $(UID) $(CONTAINER_NAME) $(CMD)
120+
121+
root-shell: up
122+
docker-compose -f ${COMPOSE_FILE_DEV} exec -u root $(CONTAINER_NAME) $(CMD)
123+
124+
down:
125+
docker-compose -f ${COMPOSE_FILE_DEV} down
126+
127+
tests: up
128+
docker-compose -f ${COMPOSE_FILE_DEV} exec -u $(UID) mapadroid-dev tox
129+
130+
unittests: up
131+
docker-compose -f ${COMPOSE_FILE_DEV} exec -u $(UID) mapadroid-dev tox -e py37
132+
133+
# Run bash within a defined tox environment
134+
# Specify a valid tox environment as such:
135+
# make shell-py37
136+
# To force a recreation of the environment, specify the RECREATE environment variable with any value
137+
# make shell-py37 RECREATE=1
138+
shell-%: up
139+
ifdef RECREATE
140+
docker-compose -f ${COMPOSE_FILE_DEV} exec -u $(UID) mapadroid-dev tox -e $* --recreate -- bash
141+
else
142+
docker-compose -f ${COMPOSE_FILE_DEV} exec -u $(UID) mapadroid-dev tox -e $* -- bash
143+
endif
144+
145+
versions:
146+
$(pip) --version
147+
$(precommit) --version
148+
$(docker) --version
149+
$(docker_compose) --version

‎docker/.dev.env

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
COMPOSE_CONVERT_WINDOWS_PATHS=1
2+
COMPOSE_FILE_DEV=docker/docker-compose-dev.yaml
3+
DOCKER_USER=dockeruser
4+
5+
LOCAL_MAD_IMAGE=local_mad_production
6+
LOCAL_MAD_DEV_IMAGE=local_mad_development
7+
8+
MYSQL_ROOT_PASSWORD=password
9+
MYSQL_DATABASE=rocketdb
10+
MYSQL_USER=rocketdb
11+
MYSQL_PASSWORD=rocketdb
12+
13+
THERAIDMAPPER_DBIP=mariadb
14+
THERAIDMAPPER_DBUSERNAME=${MYSQL_USER}
15+
THERAIDMAPPER_DBNAME=${MYSQL_DATABASE}
16+
THERAIDMAPPER_DBPASSWORD=${MYSQL_USER}

‎.dockerignore ‎docker/.dockerignore

+1
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,4 @@ temp/
66
files/
77
upload/
88
update_log.json
9+
docker/

‎Dockerfile ‎docker/Dockerfile

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
############################
22
# MAD
33
############################
4-
FROM python:3.7-slim
4+
FROM python:3.7-slim AS mad-core
55
# Working directory for the application
66
WORKDIR /usr/src/app
77

8+
89
# copy requirements only, to reduce image size and improve cache usage
910
COPY requirements.txt /usr/src/app/
1011

‎docker/Dockerfile-dev

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# Development env
2+
FROM local_mad_production:latest AS dev_test
3+
RUN pip install tox
4+
# Versions of python to install for pyenv. These are used when tox executes specific
5+
# python versions. The correct versions need to be added to tox.ini under tox/envlist
6+
ENV PYTHON_VERSIONS 3.6.0 3.7.0 3.8.0
7+
COPY requirements-test.txt /usr/src/app/
8+
ENTRYPOINT ["bash"]
9+
# Need to re-add some required dependencies for tox to compile the new envs
10+
RUN apt-get install -y --no-install-recommends \
11+
# pyenv
12+
build-essential libssl-dev zlib1g-dev libbz2-dev \
13+
libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \
14+
xz-utils tk-dev libffi-dev liblzma-dev python-openssl git \
15+
# python build
16+
libffi-dev libgdbm-dev libsqlite3-dev libssl-dev zlib1g-dev
17+
18+
# Map the user to avoid perm conflict
19+
ARG USER_NAME=dockeruser
20+
ARG UID=1000
21+
ARG GID=1000
22+
RUN groupadd -g $GID $USER_NAME; useradd -l -r -m -u $UID -g $GID $USER_NAME
23+
ENV USER $USER_NAME
24+
# Install pyenv
25+
# @TODO - How to install as a user and not root?
26+
ENV HOME=/home/dockeruser
27+
ENV PYENV_ROOT $HOME/.pyenv
28+
ENV PATH="$PYENV_ROOT/bin:$PATH"
29+
ENV PATH="$PYENV_ROOT/shims:$PATH"
30+
RUN curl -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash
31+
RUN chown -R dockeruser:dockeruser -R $HOME
32+
RUN for version in $PYTHON_VERSIONS; do \
33+
pyenv install $version; \
34+
pyenv local $version; \
35+
pip install --upgrade setuptools pip; \
36+
pyenv local --unset; \
37+
done
38+
RUN echo "pyenv local $PYTHON_VERSIONS" >> ~/.bashrc
39+
RUN pyenv local $PYTHON_VERSIONS

‎docker-compose-dev.yaml ‎docker/docker-compose-dev.yaml

+18-15
Original file line numberDiff line numberDiff line change
@@ -1,33 +1,36 @@
1-
version: "2.4"
1+
version: "3.4"
2+
3+
volumes:
4+
pre-commit-cache:
5+
name: pre-commit-cache
26

37
services:
4-
mad:
5-
image: map-a-droid
6-
build: .
8+
mapadroid-dev:
9+
image: mapadroid-dev
10+
build:
11+
context: ../.
12+
dockerfile: docker/Dockerfile-dev
713
# Use an empty entrypoint and mount local sources into container so that a simple
814
# `docker-compose run --service-ports python3 start.py` can be used to test current
915
# development in a defined environment.
1016
# `docker-compose run bash` launch a shell inside the container.
1117
entrypoint: ""
1218
volumes:
13-
- ./:/usr/src/app:rw
19+
- ../:/usr/src/app:rw
20+
- pre-commit-cache:/home/${DOCKER_USER}/.cache/pre-commit
1421
depends_on:
1522
- mariadb
1623
ports:
1724
- "5000:5000"
1825
- "8000:8000"
1926
- "8080:8080"
20-
environment:
21-
THERAIDMAPPER_DBIP: mariadb
22-
THERAIDMAPPER_DBUSERNAME: rocketdb
23-
THERAIDMAPPER_DBNAME: rocketdb
24-
THERAIDMAPPER_DBPASSWORD: rocketdb"
27+
env_file:
28+
- .dev.env
29+
tty: true
30+
command: ["bash"]
2531

2632
mariadb:
2733
image: mariadb:10.3
2834
command: mysqld --character-set-server=utf8mb4 --collation-server=utf8mb4_general_ci
29-
environment:
30-
MYSQL_DATABASE: rocketdb
31-
MYSQL_USER: rocketdb
32-
MYSQL_ROOT_PASSWORD: rocketdb
33-
MYSQL_PASSWORD: rocketdb
35+
env_file:
36+
- .dev.env

0 commit comments

Comments
 (0)
Please sign in to comment.