Skip to content

Commit 1988c89

Browse files
committed
WIP: pointclouds
1 parent d6ad351 commit 1988c89

File tree

7 files changed

+346
-11
lines changed

7 files changed

+346
-11
lines changed

backend/kangas/datatypes/datagrid.py

+3-9
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@
4343
create_columns,
4444
download_filename,
4545
expand_mask,
46-
generate_thumbnail,
4746
get_annotations_from_layers,
4847
get_labels_from_annotations,
4948
get_mask_from_annotations,
@@ -2636,14 +2635,9 @@ def _log(self, asset_id, asset_type, asset_data, metadata, row_id):
26362635
json_string = _convert_with_assets_to_json(metadata, self)
26372636
# Log to database
26382637
# If we should make a thumbnail, do it
2639-
if self.create_thumbnails and asset_type in ["Image"]:
2640-
## FIXME: check metadata "source" to retrieve from file or URL
2641-
if "annotations" in metadata and metadata["annotations"]:
2642-
annotations = json.loads(metadata["annotations"])
2643-
else:
2644-
annotations = None
2645-
asset_thumbnail = generate_thumbnail(
2646-
asset_data, annotations=annotations
2638+
if self.create_thumbnails and hasattr(ASSET_TYPE_MAP[asset_type.lower()], "generate_thumbnail"):
2639+
asset_thumbnail = ASSET_TYPE_MAP[asset_type.lower()].generate_thumbnail(
2640+
asset_data, metadata
26472641
)
26482642
else:
26492643
asset_thumbnail = None # means one hasn't been created yet

backend/kangas/datatypes/image.py

+16
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
fast_flatten,
3232
flatten,
3333
generate_image,
34+
generate_thumbnail,
3435
get_file_extension,
3536
image_to_fp,
3637
is_valid_file_path,
@@ -820,6 +821,21 @@ def add_mask_metric(
820821
return self
821822

822823

824+
@classmethod
825+
def generate_thumbnail(cls, asset_data, metadata=None):
826+
"""
827+
Args:
828+
asset_data: the raw asset data (bytes or string)
829+
metadata: the metadata dict
830+
"""
831+
## FIXME: check metadata "source" to retrieve from file or URL
832+
if metadata and "annotations" in metadata and metadata["annotations"]:
833+
annotations = json.loads(metadata["annotations"])
834+
else:
835+
annotations = None
836+
return generate_thumbnail(asset_data, annotations=annotations)
837+
838+
823839
def _image_data_to_file_like_object(
824840
image_data,
825841
file_name,

backend/kangas/datatypes/math_3d.py

+162
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
# -*- coding: utf-8 -*-
2+
# *******************************************************
3+
# ____ _ _
4+
# / ___|___ _ __ ___ ___| |_ _ __ ___ | |
5+
# | | / _ \| '_ ` _ \ / _ \ __| | '_ ` _ \| |
6+
# | |__| (_) | | | | | | __/ |_ _| | | | | | |
7+
# \____\___/|_| |_| |_|\___|\__(_)_| |_| |_|_|
8+
#
9+
# Sign up for free at https://www.comet.com
10+
# Copyright (C) 2015-2023 Comet ML INC
11+
# *******************************************************
12+
13+
import math
14+
15+
16+
def identity():
17+
"""
18+
Return matrix for identity (no transforms).
19+
"""
20+
return [
21+
[1, 0, 0, 0],
22+
[0, 1, 0, 0],
23+
[0, 0, 1, 0],
24+
[0, 0, 0, 1],
25+
]
26+
27+
28+
def rotate_x(angle):
29+
"""
30+
Return transform matrix for rotation around x axis.
31+
"""
32+
radians = angle * math.pi / 180
33+
return [
34+
[1, 0, 0, 0],
35+
[0, math.cos(radians), -math.sin(radians), 0],
36+
[0, math.sin(radians), math.cos(radians), 0],
37+
[0, 0, 0, 1],
38+
]
39+
40+
41+
def rotate_y(angle):
42+
"""
43+
Return transform matrix for rotation around y axis.
44+
"""
45+
radians = angle * math.pi / 180
46+
return [
47+
[math.cos(radians), 0, math.sin(radians), 0],
48+
[0, 1, 0, 0],
49+
[-math.sin(radians), 0, math.cos(radians), 0],
50+
[0, 0, 0, 1],
51+
]
52+
53+
54+
def rotate_z(angle):
55+
"""
56+
Return transform matrix for rotation around z axis.
57+
"""
58+
radians = angle * math.pi / 180
59+
return [
60+
[math.cos(radians), -math.sin(radians), 0, 0],
61+
[math.sin(radians), math.cos(radians), 0, 0],
62+
[0, 0, 1, 0],
63+
[0, 0, 0, 1],
64+
]
65+
66+
67+
def translate_xyz(x, y, z):
68+
"""
69+
Return transform matrix for translation (linear moving).
70+
"""
71+
return [
72+
[1, 0, 0, x],
73+
[0, 1, 0, y],
74+
[0, 0, 1, z],
75+
[0, 0, 0, 1],
76+
]
77+
78+
79+
def scale_xyz(x, y, z):
80+
"""
81+
Return transform matrix for scaling.
82+
"""
83+
return [
84+
[x, 0, 0, 0],
85+
[0, y, 0, 0],
86+
[0, 0, z, 0],
87+
[0, 0, 0, 1],
88+
]
89+
90+
91+
def matmul(a, b):
92+
"""
93+
Multiply two matrices. Written in Pure Python
94+
to avoid dependency on numpy.
95+
"""
96+
c = [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]
97+
for x in range(4):
98+
for y in range(4):
99+
acc = 0
100+
for n in range(4):
101+
acc += a[n][x] * b[y][n]
102+
c[y][x] = acc
103+
return c
104+
105+
106+
def multiply_point_by_matrix(matrix, point):
107+
"""
108+
Multiply a point by a matrix. Written in Pure Python
109+
to avoid dependency on numpy.
110+
"""
111+
return [
112+
(point[0] * matrix[0][0])
113+
+ (point[1] * matrix[0][1])
114+
+ (point[2] * matrix[0][2])
115+
+ (1 * matrix[0][3]),
116+
(point[0] * matrix[1][0])
117+
+ (point[1] * matrix[1][1])
118+
+ (point[2] * matrix[1][2])
119+
+ (1 * matrix[1][3]),
120+
(point[0] * matrix[2][0])
121+
+ (point[1] * matrix[2][1])
122+
+ (point[2] * matrix[2][2])
123+
+ (1 * matrix[2][3]),
124+
]
125+
126+
127+
def point_to_canvas(size, point, z=False):
128+
"""
129+
Convert to screen coordinates (flip horizontally)
130+
Only return the first two values [x, y] of point
131+
"""
132+
if z:
133+
return [int(size[0] - point[0]), int(point[1]), point[2]]
134+
else:
135+
return [int(size[0] - point[0]), int(point[1])]
136+
137+
138+
def draw_line(size, canvas, transform, a, b, color):
139+
"""
140+
Draw a line on the canvas given two points and transform.
141+
"""
142+
ta = point_to_canvas(size, multiply_point_by_matrix(transform, a))
143+
tb = point_to_canvas(size, multiply_point_by_matrix(transform, b))
144+
canvas.line(ta + tb, fill=color)
145+
146+
147+
def draw_point(size, canvas, transform, point, color):
148+
"""
149+
Draw a point on the canvas given the transform.
150+
"""
151+
p = point_to_canvas(size, multiply_point_by_matrix(transform, point))
152+
canvas.point(p, fill=color)
153+
154+
155+
def draw_point_fake(size, fcanvas, transform, point, color):
156+
"""
157+
Draw a point on the canvas given the transform.
158+
"""
159+
p = point_to_canvas(size, multiply_point_by_matrix(transform, point), z=True)
160+
location = fcanvas[(p[0], p[1])]
161+
if location is None or location["z"] < p[2]:
162+
fcanvas[(p[0], p[1])] = {"z": p[2], "color": color}

backend/kangas/datatypes/pointcloud.py

+63
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,12 @@
1313

1414
import logging
1515
import json
16+
import tempfile
17+
import random
1618

1719
from .base import Asset
20+
from .thumbnail import create_thumbnail
21+
from .utils import image_to_fp
1822

1923
LOGGER = logging.getLogger(__name__)
2024

@@ -41,12 +45,71 @@ def __init__(
4145
self._unserialize = unserialize
4246
return
4347

48+
points = points if points is not None else []
49+
boxes = boxes if boxes is not None else []
50+
51+
min_max_x = [float("inf"), float("-inf")]
52+
min_max_y = [float("inf"), float("-inf")]
53+
min_max_z = [float("inf"), float("-inf")]
54+
55+
for point in points:
56+
min_max_x = min(point[0], min_max_x[0]), max(point[0], min_max_x[1])
57+
min_max_y = min(point[1], min_max_y[0]), max(point[1], min_max_y[1])
58+
min_max_z = min(point[2], min_max_z[0]), max(point[2], min_max_z[1])
59+
60+
random.shuffle(points)
61+
62+
for box in boxes:
63+
for segment_points in box["segments"]:
64+
for point in segment_points:
65+
min_max_x = min(point[0], min_max_x[0]), max(
66+
point[0], min_max_x[1]
67+
)
68+
min_max_y = min(point[1], min_max_y[0]), max(
69+
point[1], min_max_y[1]
70+
)
71+
min_max_z = min(point[2], min_max_z[0]), max(
72+
point[2], min_max_z[1]
73+
)
74+
4475
self.metadata["scene_name"] = scene_name
4576
self.metadata["step"] = step
77+
self.metadata["min_max_x"] = min_max_x
78+
self.metadata["min_max_y"] = min_max_y
79+
self.metadata["min_max_z"] = min_max_z
4680

4781
self.asset_data = json.dumps(
4882
{
4983
"points": points,
5084
"boxes": boxes,
5185
}
5286
)
87+
88+
@classmethod
89+
def generate_thumbnail(cls, asset_data_raw, metadata=None):
90+
"""
91+
Args:
92+
asset_data_raw: the raw asset data (bytes or string)
93+
metadata: the metadata dict
94+
"""
95+
asset_data = json.loads(asset_data_raw)
96+
points = asset_data["points"]
97+
boxes = asset_data["boxes"]
98+
min_max_x = metadata["min_max_x"]
99+
min_max_y = metadata["min_max_y"]
100+
min_max_z = metadata["min_max_z"]
101+
102+
thumbnail_image = create_thumbnail(
103+
points,
104+
boxes,
105+
45,
106+
0,
107+
45,
108+
min_max_x,
109+
min_max_y,
110+
min_max_z,
111+
)
112+
113+
fp = image_to_fp(thumbnail_image, "png")
114+
image_data = fp.read()
115+
return image_data

backend/kangas/datatypes/thumbnail.py

+93
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
# -*- coding: utf-8 -*-
2+
# *******************************************************
3+
# ____ _ _
4+
# / ___|___ _ __ ___ ___| |_ _ __ ___ | |
5+
# | | / _ \| '_ ` _ \ / _ \ __| | '_ ` _ \| |
6+
# | |__| (_) | | | | | | __/ |_ _| | | | | | |
7+
# \____\___/|_| |_| |_|\___|\__(_)_| |_| |_|_|
8+
#
9+
# Sign up for free at https://www.comet.com
10+
# Copyright (C) 2015-2023 Comet ML INC
11+
# *******************************************************
12+
13+
import json
14+
import logging
15+
import tempfile
16+
from collections import defaultdict
17+
18+
from .._typing import Any
19+
from . import math_3d
20+
21+
LOGGER = logging.getLogger(__name__)
22+
23+
24+
def create_thumbnail(
25+
points, boxes, x, y, z, min_max_x, min_max_y, min_max_z
26+
) -> Any:
27+
try:
28+
from PIL import Image, ImageDraw
29+
except ImportError:
30+
LOGGER.error(
31+
"The Python library Pillow is required to generate a 3D Cloud thumbnail"
32+
)
33+
return None
34+
35+
size = (250, 250)
36+
background_color = (51, 51, 77)
37+
38+
image = Image.new("RGB", size, background_color)
39+
canvas = ImageDraw.Draw(image)
40+
41+
midpoint = [
42+
(min_max_x[0] + min_max_x[1]) / 2,
43+
(min_max_y[0] + min_max_y[1]) / 2,
44+
(min_max_z[0] + min_max_z[1]) / 2,
45+
]
46+
47+
x_range = abs(min_max_x[0] - min_max_x[1])
48+
y_range = abs(min_max_y[0] - min_max_y[1])
49+
50+
scale = min(
51+
size[0] / (x_range if x_range != 0 else 1),
52+
size[1] / (y_range if y_range != 0 else 1),
53+
)
54+
transform = math_3d.identity()
55+
transform = math_3d.matmul(
56+
transform, math_3d.translate_xyz(*[-n for n in midpoint])
57+
)
58+
transform = math_3d.matmul(transform, math_3d.rotate_z(z))
59+
transform = math_3d.matmul(transform, math_3d.rotate_x(x))
60+
transform = math_3d.matmul(transform, math_3d.rotate_y(y))
61+
transform = math_3d.matmul(transform, math_3d.scale_xyz(scale, scale, scale))
62+
transform = math_3d.matmul(
63+
transform, math_3d.translate_xyz(size[0] / 2, size[1] / 2, 0)
64+
)
65+
66+
fcanvas = defaultdict(lambda: None)
67+
for data in points:
68+
point = data[:3]
69+
if len(data) > 3:
70+
color = tuple([int(round(c)) for c in data[3:]])
71+
else:
72+
color = (255, 255, 255)
73+
math_3d.draw_point_fake(size, fcanvas, transform, point, color)
74+
75+
if fcanvas:
76+
for x, y in fcanvas:
77+
color = fcanvas[(x, y)]["color"]
78+
canvas.point((x, y), fill=color)
79+
80+
for data in boxes:
81+
if "color" in data and data["color"]:
82+
color = tuple(data["color"])
83+
else:
84+
color = (255, 255, 255)
85+
for points in data["segments"]:
86+
point1 = points[0]
87+
for point2 in points[1:]:
88+
math_3d.draw_line(
89+
size, canvas, transform, point1, point2, color
90+
)
91+
point1 = point2
92+
93+
return image

0 commit comments

Comments
 (0)