diff --git a/build/camera_module/build/lib/camera_module/camera_module.py b/build/camera_module/build/lib/camera_module/camera_module.py index d7a9f19..bb81bb6 100644 --- a/build/camera_module/build/lib/camera_module/camera_module.py +++ b/build/camera_module/build/lib/camera_module/camera_module.py @@ -11,27 +11,27 @@ from std_msgs.msg import String import rclpy + + class CamPublisher(ThreadedNode): - def __init__(self): + def __init__(self, rknn, cap): super().__init__('camera_module', default_rate=5.0) self.string_pub = self.create_publisher(String, 'camera_module/cam_topic', 10) self.image_pub = self.create_publisher(CompressedImage, 'camera_module/compressed', 10) - - # Try multiple device indices - self.cap = None - for i in range(5): - cap = cv2.VideoCapture(i) - if cap.isOpened(): - self.cap = cap - self.get_logger().info(f'Opened webcam at index {i}') - break - + self.rknn = rknn + self.cap = cap + self.latest_frame = None + self.latest_faces = [] + self.model_size = (320, 320) + self.priors = PriorBox(self.model_size) + self.prev_time = time.time() + if self.cap is None: self.get_logger().error('Failed to open any webcam') # Lower resolution - self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, 320) - self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 240) + #self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, 320) + #self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 240) # Faster JPEG encoding @@ -50,30 +50,161 @@ class CamPublisher(ThreadedNode): self.get_logger().error('No webcam available') return - ret, frame = self.cap.read() - if not ret or frame is None: - self.get_logger().error('Camera read failed') + ret, frame = cap.read() + if not ret: + return + img_height, img_width, _ = frame.shape + letterbox_img, aspect_ratio, offset_x, offset_y = letterbox_resize(frame, model_size, 114) + infer_img = np.expand_dims(letterbox_img.astype(np.uint8), axis=0) + + outputs = rknn.inference(inputs=[infer_img]) + if outputs is None: return - try: - msg = CompressedImage() - msg.header.stamp = self.get_clock().now().to_msg() - msg.format = 'jpeg' - msg.data = cv2.imencode('.jpg', frame, [int(cv2.IMWRITE_JPEG_QUALITY), 50])[1].tobytes() - #msg.data = cv2.imencode('.jpg', frame)[1].tobytes() - self.image_pub.publish(msg) - self.get_logger().info('Published webcam frame') - except Exception as e: - self.get_logger().error(f'Failed to convert/publish image: {e}') + loc, conf, landms = outputs + boxes = box_decode(loc.squeeze(0), priors) + boxes *= np.array([model_size[1], model_size[0], model_size[1], model_size[0]]) + boxes[:, 0::2] = np.clip((boxes[:, 0::2] - offset_x) / aspect_ratio, 0, img_width) + boxes[:, 1::2] = np.clip((boxes[:, 1::2] - offset_y) / aspect_ratio, 0, img_height) + + scores = conf.squeeze(0)[:, 1] + landms = decode_landm(landms.squeeze(0), priors) + landms *= np.tile(np.array([model_size[1], model_size[0]]), 5) + landms[:, 0::2] = np.clip((landms[:, 0::2] - offset_x) / aspect_ratio, 0, img_width) + landms[:, 1::2] = np.clip((landms[:, 1::2] - offset_y) / aspect_ratio, 0, img_height) + + inds = np.where(scores > 0.2)[0] + boxes, landms, scores = boxes[inds], landms[inds], scores[inds] + order = scores.argsort()[::-1] + boxes, landms, scores = boxes[order], landms[order], scores[order] + + dets = np.hstack((boxes, scores[:, np.newaxis])).astype(np.float32) + keep = nms(dets, 0.5) + dets, landms = dets[keep], landms[keep] + + face_data = [] + frame_center = np.array([img_width / 2, img_height / 2]) + + for data, landmark in zip(dets, landms): + if data[4] < 0.6: + continue + x1, y1, x2, y2 = map(int, data[:4]) + conf = data[4] + box_center = np.array([(x1 + x2) / 2, (y1 + y2) / 2]) + offset = box_center - frame_center + face_data.append({ + "box": [x1, y1, x2, y2], + "confidence": float(conf), + "offset_from_center": { + "x": float(offset[0]), + "y": float(offset[1]) + } + }) + cv2.rectangle(frame, (x1, y1), (x2, y2), (0, 0, 255), 2) + cv2.putText(frame, f'{conf:.4f}', (x1, y1 + 12), cv2.FONT_HERSHEY_DUPLEX, 0.5, (255, 255, 255)) + for j in range(5): + lx, ly = map(int, landmark[j*2:j*2+2]) + cv2.circle(frame, (lx, ly), 1, (0, 255, 255), 2) + + cv2.putText(frame, f'FPS: {fps:.2f}', (10, 30), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2) + if len(face_data) > 0: + print(face_data) + ret, buffer = cv2.imencode('.jpg', frame) + if ret: + latest_frame = buffer.tobytes() + latest_faces = face_data def destroy_node(self): if self.cap: self.cap.release() super().destroy_node() + +# --- RetinaFace Utilities --- +def letterbox_resize(image, size, bg_color): + target_width, target_height = size + image_height, image_width, _ = image.shape + aspect_ratio = min(target_width / image_width, target_height / image_height) + new_width = int(image_width * aspect_ratio) + new_height = int(image_height * aspect_ratio) + image = cv2.resize(image, (new_width, new_height), interpolation=cv2.INTER_AREA) + result_image = np.ones((target_height, target_width, 3), dtype=np.uint8) * bg_color + offset_x = (target_width - new_width) // 2 + offset_y = (target_height - new_height) // 2 + result_image[offset_y:offset_y + new_height, offset_x:offset_x + new_width] = image + return result_image, aspect_ratio, offset_x, offset_y + +def PriorBox(image_size): + anchors = [] + min_sizes = [[16, 32], [64, 128], [256, 512]] + steps = [8, 16, 32] + feature_maps = [[ceil(image_size[0] / step), ceil(image_size[1] / step)] for step in steps] + for k, f in enumerate(feature_maps): + min_sizes_ = min_sizes[k] + for i, j in product(range(f[0]), range(f[1])): + for min_size in min_sizes_: + s_kx = min_size / image_size[1] + s_ky = min_size / image_size[0] + dense_cx = [x * steps[k] / image_size[1] for x in [j + 0.5]] + dense_cy = [y * steps[k] / image_size[0] for y in [i + 0.5]] + for cy, cx in product(dense_cy, dense_cx): + anchors += [cx, cy, s_kx, s_ky] + return np.array(anchors).reshape(-1, 4) + +def box_decode(loc, priors): + variances = [0.1, 0.2] + boxes = np.concatenate(( + priors[:, :2] + loc[:, :2] * variances[0] * priors[:, 2:], + priors[:, 2:] * np.exp(loc[:, 2:] * variances[1])), axis=1) + boxes[:, :2] -= boxes[:, 2:] / 2 + boxes[:, 2:] += boxes[:, :2] + return boxes + +def decode_landm(pre, priors): + variances = [0.1, 0.2] + landmarks = np.concatenate(( + priors[:, :2] + pre[:, 0:2] * variances[0] * priors[:, 2:], + priors[:, :2] + pre[:, 2:4] * variances[0] * priors[:, 2:], + priors[:, :2] + pre[:, 4:6] * variances[0] * priors[:, 2:], + priors[:, :2] + pre[:, 6:8] * variances[0] * priors[:, 2:], + priors[:, :2] + pre[:, 8:10] * variances[0] * priors[:, 2:] + ), axis=1) + return landmarks + +def nms(dets, thresh): + x1, y1, x2, y2, scores = dets[:, 0], dets[:, 1], dets[:, 2], dets[:, 3], dets[:, 4] + areas = (x2 - x1 + 1) * (y2 - y1 + 1) + order = scores.argsort()[::-1] + keep = [] + while order.size > 0: + i = order[0] + keep.append(i) + xx1 = np.maximum(x1[i], x1[order[1:]]) + yy1 = np.maximum(y1[i], y1[order[1:]]) + xx2 = np.minimum(x2[i], x2[order[1:]]) + yy2 = np.minimum(y2[i], y2[order[1:]]) + w = np.maximum(0.0, xx2 - xx1 + 1) + h = np.maximum(0.0, yy2 - yy1 + 1) + inter = w * h + ovr = inter / (areas[i] + areas[order[1:]] - inter) + inds = np.where(ovr <= thresh)[0] + order = order[inds + 1] + return keep + +# --- RKNN Initialization --- +rknn = RKNNLite() +rknn.load_rknn('./RetinaFace.rknn') +rknn.init_runtime() + +# --- Shared State --- +latest_frame = None +latest_faces = [] +cap = cv2.VideoCapture(0) + def main(): rclpy.init() - node = CamPublisher() + node = CamPublisher(rknn, cap) rclpy.spin(node) node.destroy_node() rclpy.shutdown() diff --git a/build/camera_module/colcon_command_prefix_setup_py.sh.env b/build/camera_module/colcon_command_prefix_setup_py.sh.env index b6b9200..131af4a 100644 --- a/build/camera_module/colcon_command_prefix_setup_py.sh.env +++ b/build/camera_module/colcon_command_prefix_setup_py.sh.env @@ -16,7 +16,6 @@ LESSCLOSE=/usr/bin/lesspipe %s %s LESSOPEN=| /usr/bin/lesspipe %s LOGNAME=jake LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=00:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.avif=01;35:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.webp=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:*~=00;90:*#=00;90:*.bak=00;90:*.crdownload=00;90:*.dpkg-dist=00;90:*.dpkg-new=00;90:*.dpkg-old=00;90:*.dpkg-tmp=00;90:*.old=00;90:*.orig=00;90:*.part=00;90:*.rej=00;90:*.rpmnew=00;90:*.rpmorig=00;90:*.rpmsave=00;90:*.swp=00;90:*.tmp=00;90:*.ucf-dist=00;90:*.ucf-new=00;90:*.ucf-old=00;90: -MOTD_SHOWN=update-motd NAME=DESKTOP-UFLG41E OLDPWD=/home/jake/ros2_ws PATH=/home/jake/.vscode-server/bin/7d842fb85a0275a4a8e4d7e040d2625abbf7f084/bin/remote-cli:/home/jake/.local/bin:/home/jake/miniconda3/condabin:/opt/ros/kilted/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/usr/lib/wsl/lib:/mnt/c/Windows/system32:/mnt/c/Windows:/mnt/c/Windows/System32/Wbem:/mnt/c/Windows/System32/WindowsPowerShell/v1.0/:/mnt/c/Windows/System32/OpenSSH/:/mnt/c/Program Files/Docker/Docker/resources/bin:/mnt/c/Program Files/usbipd-win/:/mnt/c/Users/jake/AppData/Local/Microsoft/WindowsApps:/mnt/c/Users/jake/AppData/Local/Programs/Microsoft VS Code/bin:/snap/bin:/home/jake/.vscode-server/data/User/globalStorage/github.copilot-chat/debugCommand @@ -37,13 +36,13 @@ VSCODE_GIT_ASKPASS_EXTRA_ARGS= VSCODE_GIT_ASKPASS_MAIN=/home/jake/.vscode-server/bin/7d842fb85a0275a4a8e4d7e040d2625abbf7f084/extensions/git/dist/askpass-main.js VSCODE_GIT_ASKPASS_NODE=/home/jake/.vscode-server/bin/7d842fb85a0275a4a8e4d7e040d2625abbf7f084/node VSCODE_GIT_IPC_HANDLE=/run/user/1001/vscode-git-1d6e8e65c1.sock -VSCODE_IPC_HOOK_CLI=/run/user/1001/vscode-ipc-a80506a6-314e-4f1a-bfb0-04fa92945475.sock +VSCODE_IPC_HOOK_CLI=/run/user/1001/vscode-ipc-07b86bca-e4f9-4c18-ba3e-d1abb3d5b6bc.sock VSCODE_PYTHON_AUTOACTIVATE_GUARD=1 WAYLAND_DISPLAY=wayland-0 WSL2_GUI_APPS_ENABLED=1 WSLENV=VSCODE_WSL_EXT_LOCATION/up WSL_DISTRO_NAME=Ubuntu-24.04 -WSL_INTEROP=/run/WSL/170769_interop +WSL_INTEROP=/run/WSL/440_interop XDG_DATA_DIRS=/usr/local/share:/usr/share:/var/lib/snapd/desktop XDG_RUNTIME_DIR=/run/user/1001/ _=/usr/bin/colcon diff --git a/build/camera_module/prefix_override/__pycache__/sitecustomize.cpython-312.pyc b/build/camera_module/prefix_override/__pycache__/sitecustomize.cpython-312.pyc index cdd69df..32cecdc 100644 Binary files a/build/camera_module/prefix_override/__pycache__/sitecustomize.cpython-312.pyc and b/build/camera_module/prefix_override/__pycache__/sitecustomize.cpython-312.pyc differ diff --git a/index.html b/index.html index ae3c564..19b9b25 100644 --- a/index.html +++ b/index.html @@ -58,7 +58,7 @@ Camera Feed