Skip to content

Commit 799f7c6

Browse files
Iteron-devIteron-dev
committed
Refactor&Build: Update Dockerfile and implement sandbox caching
1 parent de5d478 commit 799f7c6

File tree

6 files changed

+250
-79
lines changed

6 files changed

+250
-79
lines changed

‎Dockerfile

+24-13
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM python:3.10
1+
FROM python:3.10 AS base
22

33
ENV PYTHONUNBUFFERED 1
44

@@ -66,23 +66,34 @@ RUN pip3 install -r requirements_static.txt --user
6666

6767
COPY --chown=oioioi:oioioi . /sio2/oioioi
6868

69-
70-
ENV OIOIOI_DB_ENGINE 'django.db.backends.postgresql'
71-
ENV RABBITMQ_HOST 'broker'
72-
ENV RABBITMQ_PORT '5672'
73-
ENV RABBITMQ_USER 'oioioi'
74-
ENV RABBITMQ_PASSWORD 'oioioi'
75-
ENV FILETRACKER_LISTEN_ADDR '0.0.0.0'
76-
ENV FILETRACKER_LISTEN_PORT '9999'
77-
ENV FILETRACKER_URL 'http://web:9999'
78-
7969
RUN oioioi-create-config /sio2/deployment
8070

8171
WORKDIR /sio2/deployment
8272

8373
RUN mkdir -p /sio2/deployment/logs/{supervisor,runserver}
8474

85-
# Download sandboxes
75+
FROM python:3.10 AS development-sandboxes
76+
77+
ENV DOWNLOAD_DIR=/sio2/sandboxes
78+
ENV MANIFEST_URL=https://downloads.sio2project.mimuw.edu.pl/sandboxes/Manifest
79+
80+
RUN apt-get update && \
81+
apt-get install -y curl wget bash && \
82+
apt-get clean
83+
84+
ADD $MANIFEST_URL /sio2/Manifest
85+
86+
COPY download_sandboxes.sh /download_sandboxes.sh
87+
RUN chmod +x /download_sandboxes.sh
88+
89+
RUN ./download_sandboxes.sh -q -y -d $DOWNLOAD_DIR -m $MANIFEST_URL
90+
91+
FROM base AS development
92+
93+
COPY --from=development-sandboxes /sio2/sandboxes /sio2/sandboxes
94+
RUN chmod +x /sio2/oioioi/download_sandboxes.sh
95+
8696
RUN ./manage.py supervisor > /dev/null --daemonize --nolaunch=uwsgi && \
87-
./manage.py download_sandboxes -q -y -c /sio2/sandboxes && \
97+
/sio2/oioioi/wait-for-it.sh -t 60 "127.0.0.1:9999" && \
98+
./manage.py download_sandboxes -q -y -c /sio2/sandboxes -p /sio2/oioioi/download_sandboxes.sh && \
8899
./manage.py supervisor stop all

‎docker-compose-dev.yml

+12
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,22 @@ services:
1414
build:
1515
context: .
1616
dockerfile: Dockerfile
17+
target: development
1718
args:
1819
- "oioioi_uid=${OIOIOI_UID}"
1920
extra_hosts:
2021
- "web:127.0.0.1"
22+
environment:
23+
OIOIOI_DB_ENGINE: 'django.db.backends.postgresql'
24+
RABBITMQ_HOST: 'broker'
25+
RABBITMQ_PORT: '5672'
26+
RABBITMQ_USER: 'oioioi'
27+
RABBITMQ_PASSWORD: 'oioioi'
28+
FILETRACKER_LISTEN_ADDR: '0.0.0.0'
29+
FILETRACKER_LISTEN_PORT: '9999'
30+
FILETRACKER_URL: 'http://web:9999'
31+
DATABASE_HOST: 'db'
32+
DATABASE_PORT: '5432'
2133
ports:
2234
# web server
2335
- "8000:8000"

‎docker-compose.yml

+11
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,17 @@ services:
1111
web:
1212
image: sio2project/oioioi:$OIOIOI_VERSION
1313
command: ["/sio2/oioioi/oioioi_init.sh"]
14+
environment:
15+
OIOIOI_DB_ENGINE: 'django.db.backends.postgresql'
16+
RABBITMQ_HOST: 'broker'
17+
RABBITMQ_PORT: '5672'
18+
RABBITMQ_USER: 'oioioi'
19+
RABBITMQ_PASSWORD: 'oioioi'
20+
FILETRACKER_LISTEN_ADDR: '0.0.0.0'
21+
FILETRACKER_LISTEN_PORT: '9999'
22+
FILETRACKER_URL: 'http://web:9999'
23+
DATABASE_HOST: 'db'
24+
DATABASE_PORT: '5432'
1425
ports:
1526
- "8000:8000"
1627
stop_grace_period: 3m

‎download_sandboxes.sh

+166
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
#!/bin/bash
2+
3+
DEFAULT_MANIFEST_URL="https://downloads.sio2project.mimuw.edu.pl/sandboxes/Manifest"
4+
DEFAULT_DOWNLOAD_DIR="sandboxes-download"
5+
DEFAULT_WGET="wget"
6+
QUIET=false
7+
AGREE_LICENSE=false
8+
9+
echoerr() { echo "$@" 1>&2; }
10+
11+
usage() {
12+
echo "Usage: $0 [options] [sandbox1 sandbox2 ...]"
13+
echo ""
14+
echo "Options:"
15+
echo " -m, --manifest URL Specifies URL with the Manifest file listing available sandboxes (default: $DEFAULT_MANIFEST_URL)"
16+
echo " -d, --download-dir DIR Specify the download directory (default: $DEFAULT_DOWNLOAD_DIR)"
17+
echo " -c, --cache-dir DIR Load cached sandboxes from a local directory (default: None)"
18+
echo " --wget PATH Specify the wget binary to use (default: $DEFAULT_WGET)"
19+
echo " -y, --yes Enabling this options means that you agree to the license terms and conditions, so no license prompt will be displayed"
20+
echo " -q, --quiet Disables wget interactive progress bars"
21+
echo " -h, --help Display this help message"
22+
exit 1
23+
}
24+
25+
while [[ $# -gt 0 ]]; do
26+
case "$1" in
27+
-m|--manifest)
28+
MANIFEST_URL="$2"
29+
shift 2
30+
;;
31+
-d|--download-dir)
32+
DOWNLOAD_DIR="$2"
33+
shift 2
34+
;;
35+
-c|--cache-dir)
36+
CACHE_DIR="$2"
37+
shift 2
38+
;;
39+
--wget)
40+
WGET_CMD="$2"
41+
shift 2
42+
;;
43+
-y|--yes)
44+
AGREE_LICENSE=true
45+
shift
46+
;;
47+
-q|--quiet)
48+
QUIET=true
49+
shift
50+
;;
51+
-h|--help)
52+
usage
53+
;;
54+
--)
55+
shift
56+
break
57+
;;
58+
-*)
59+
echoerr "Unknown argument: $1"
60+
usage
61+
;;
62+
*)
63+
break
64+
;;
65+
esac
66+
done
67+
68+
MANIFEST_URL="${MANIFEST_URL:-$DEFAULT_MANIFEST_URL}"
69+
DOWNLOAD_DIR="${DOWNLOAD_DIR:-$DEFAULT_DOWNLOAD_DIR}"
70+
WGET_CMD="${WGET_CMD:-$DEFAULT_WGET}"
71+
72+
SANDBOXES=("$@")
73+
74+
75+
if ! MANIFEST_CONTENT=$(curl -fsSL "$MANIFEST_URL"); then
76+
echoerr "Error: Unable to download manifest from $MANIFEST_URL"
77+
exit 1
78+
fi
79+
80+
IFS=$'\n' read -d '' -r -a MANIFEST <<< "$MANIFEST_CONTENT"
81+
82+
83+
BASE_URL=$(dirname "$MANIFEST_URL")/
84+
LICENSE_URL="${BASE_URL}LICENSE"
85+
86+
LICENSE_CONTENT=$(curl -fsSL "$LICENSE_URL")
87+
LICENSE_STATUS=$?
88+
89+
if [[ $LICENSE_STATUS -eq 0 ]]; then
90+
if ! $AGREE_LICENSE; then
91+
echoerr ""
92+
echoerr "The sandboxes are accompanied with a license:"
93+
echoerr "$LICENSE_CONTENT"
94+
while true; do
95+
read -rp "Do you accept the license? (yes/no): " yn
96+
case "$yn" in
97+
yes ) break;;
98+
no ) echoerr "License not accepted. Exiting..."; exit 1;;
99+
* ) echoerr 'Please enter either "yes" or "no".';;
100+
esac
101+
done
102+
fi
103+
elif [[ $LICENSE_STATUS -ne 22 ]]; then
104+
echoerr "Error: Unable to download LICENSE from $LICENSE_URL"
105+
exit 1
106+
fi
107+
108+
if [[ ${#SANDBOXES[@]} -eq 0 ]]; then
109+
SANDBOXES=("${MANIFEST[@]}")
110+
fi
111+
112+
113+
URLS=()
114+
for SANDBOX in "${SANDBOXES[@]}"; do
115+
found=false
116+
for item in "${MANIFEST[@]}"; do
117+
if [[ "$item" == "$SANDBOX" ]]; then
118+
found=true
119+
break
120+
fi
121+
done
122+
123+
if [[ $found == false ]]; then
124+
echoerr "Error: Sandbox '$SANDBOX' not available (not in Manifest)"
125+
exit 1
126+
fi
127+
128+
echo "$SANDBOX";
129+
130+
BASENAME="${SANDBOX}.tar.gz"
131+
132+
if [[ -n "$CACHE_DIR" && -f "$CACHE_DIR/$BASENAME" ]]; then
133+
continue
134+
fi
135+
136+
URL="${BASE_URL}${BASENAME}"
137+
URLS+=("$URL")
138+
done
139+
140+
if [[ ! -d "$DOWNLOAD_DIR" ]]; then
141+
if ! mkdir -p "$DOWNLOAD_DIR"; then
142+
echoerr "Error: Unable to create download directory '$DOWNLOAD_DIR'"
143+
exit 1
144+
fi
145+
fi
146+
147+
if ! command -v "$WGET_CMD" &> /dev/null; then
148+
echoerr "Error: '$WGET_CMD' is not installed or not in PATH."
149+
exit 1
150+
fi
151+
152+
WGET_OPTIONS=("--no-check-certificate")
153+
if $QUIET; then
154+
WGET_OPTIONS+=("-nv")
155+
fi
156+
157+
for URL in "${URLS[@]}"; do
158+
BASENAME=$(basename "$URL")
159+
OUTPUT_PATH="$DOWNLOAD_DIR/$BASENAME"
160+
if ! "$WGET_CMD" "${WGET_OPTIONS[@]}" -O "$OUTPUT_PATH" "$URL"; then
161+
echoerr "Error: Failed to download $BASENAME"
162+
exit 1
163+
fi
164+
done
165+
166+
exit 0

0 commit comments

Comments
 (0)