{ "cells": [ { "cell_type": "markdown", "id": "c3e65b07", "metadata": {}, "source": [ "# Postprocessing of upper and bottom lines\n", "Fix the problem when the bottom line disappear and fix the upper line" ] }, { "cell_type": "code", "execution_count": 100, "id": "07e4eaf6", "metadata": {}, "outputs": [], "source": [ "import cv2\n", "from pathlib import Path\n", "import numpy as np\n", "import pandas as pd" ] }, { "cell_type": "markdown", "id": "79556f0b", "metadata": {}, "source": [ "Import video" ] }, { "cell_type": "code", "execution_count": 101, "id": "922828f3", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Opened: True, FPS: 59.94005994005994, Total Frames: 227.0\n" ] } ], "source": [ "video_number = \"3\"\n", "# Define the relative path to the video file\n", "notebook_dir = Path().resolve()\n", "project_root = notebook_dir.parent.parent\n", "video_path = (\n", " project_root\n", " / \"data\"\n", " / f\"recording_{video_number}\"\n", " / f\"Recording_{video_number}.mp4\"\n", ")\n", "video_path = str(video_path)\n", "\n", "# Load the video\n", "cap = cv2.VideoCapture(video_path)\n", "\n", "# Check\n", "print(\n", " f\"Opened: {cap.isOpened()}, FPS: {cap.get(cv2.CAP_PROP_FPS)}, Total Frames: {cap.get(cv2.CAP_PROP_FRAME_COUNT)}\"\n", ")" ] }, { "cell_type": "markdown", "id": "b139c2ed", "metadata": {}, "source": [ "Import points" ] }, { "cell_type": "code", "execution_count": 102, "id": "c5eae99c", "metadata": {}, "outputs": [], "source": [ "# Define the path to the CSV file\n", "input_data_path = (\n", " project_root\n", " / \"notebook\"\n", " / \"lane_detection\"\n", " / \"intermediate_data\"\n", " / \"lane_points\"\n", " / f\"lane_points_raw_{video_number}.csv\"\n", ")\n", "\n", "# Load the CSV file into a DataFrame\n", "points_df = pd.read_csv(input_data_path)" ] }, { "cell_type": "markdown", "id": "921f5768", "metadata": {}, "source": [ "Function to determine when the bottom line disappear" ] }, { "cell_type": "code", "execution_count": 103, "id": "1d225134", "metadata": {}, "outputs": [], "source": [ "\"\"\"if both are above return them\n", "if one is above and the other is below return the one that is above and the other one \n", " compute as the intersection of the lateral line with a line with teh same slope of the first frame that pass through the correct point\n", "if both are below return None, None -> fix the prolem later\n", "\"\"\"\n", "\n", "\n", "def is_disappeared(bl_0, br_0, bl, br, tr, tl, max_y, threshold=0.99):\n", " if (\n", " bl[1] < max_y * threshold and br[1] < max_y * threshold\n", " ): # both points are above the threshold -> keep them\n", " return bl, br\n", " return None, None" ] }, { "cell_type": "markdown", "id": "ee2f3331", "metadata": {}, "source": [ "Compute the intersection point" ] }, { "cell_type": "code", "execution_count": 104, "id": "646eace5", "metadata": {}, "outputs": [], "source": [ "def get_intersection(p1, p2, p3, p4):\n", " # Calculate the direction vectors of the lines\n", " dx1, dy1 = p2[0] - p1[0], p2[1] - p1[1]\n", " dx2, dy2 = p4[0] - p3[0], p4[1] - p3[1]\n", "\n", " # Calculate the determinant to check if lines are parallel\n", " determinant = dx1 * dy2 - dy1 * dx2\n", " if determinant == 0:\n", " return None # Lines are parallel\n", "\n", " # Calculate the intersection point using the parametric equations of the lines\n", " t = ((p3[0] - p1[0]) * dy2 - (p3[1] - p1[1]) * dx2) / determinant\n", " intersection = (p1[0] + t * dx1, p1[1] + t * dy1)\n", "\n", " return intersection" ] }, { "cell_type": "markdown", "id": "166d3776", "metadata": {}, "source": [ "Postprocessing upper line" ] }, { "cell_type": "code", "execution_count": 105, "id": "2510b92d", "metadata": {}, "outputs": [], "source": [ "def postprocessing_upper(points_df, bottom_y_distances):\n", " df = points_df[[\"Frame\", \"up_left_y\"]].copy()\n", " df_length = len(df)\n", " # print('last 100 df:', df['up_left_y'].values[-100:])\n", "\n", " distances = np.diff(df[\"up_left_y\"].values)\n", " # print('length:', len(distances), 'length df:', len(df))\n", "\n", " mean_distance = np.mean(distances)\n", " std_distance = np.std(distances)\n", " print(\"Mean distance:\", mean_distance)\n", " print(\"Std distance:\", std_distance)\n", "\n", " threshold = std_distance\n", " num_outliers = 100\n", "\n", " while num_outliers > 0:\n", " filtered_distances = np.where(np.abs(distances) > threshold, np.nan, distances)\n", " # print('Indexes with NaN in filtered_distances:', np.where(np.isnan(filtered_distances))[0])\n", " num_outliers = np.sum(np.isnan(filtered_distances))\n", "\n", " # # remove from df the rows with nan in filtered_distances\n", " # df = df.iloc[np.where(~np.isnan(filtered_distances))[0]]\n", "\n", " # Trova gli indici dei NaN\n", " nan_indices = np.where(np.isnan(filtered_distances))[0]\n", " # Calcola gli indici delle righe successive (i+1), evitando out-of-bounds\n", " next_indices = nan_indices + 1\n", " next_indices = next_indices[\n", " next_indices < len(df)\n", " ] # assicurati di restare nel range\n", " # Rimuovi solo le righe successive\n", " df = df.drop(index=df.index[next_indices]).reset_index(drop=True)\n", "\n", " # compute angain the distances\n", " distances = np.diff(df[\"up_left_y\"].values)\n", " # print('last 100 distances:', distances[-100:])\n", "\n", " # interpolate to found the missing values\n", " df[\"up_left_y\"] = df[\"up_left_y\"].interpolate(\n", " method=\"linear\", limit_direction=\"both\"\n", " )\n", " # print('last 100 df:', df['up_left_y'].values[-100:])\n", " # print('length:', len(df))\n", "\n", " # fill the remaining values at the end of the df with estimated values\n", " for i in range(len(df), df_length):\n", " if i >= len(df):\n", " df = pd.concat(\n", " [\n", " df,\n", " pd.DataFrame(\n", " {\n", " \"Frame\": [i],\n", " \"up_left_y\": [\n", " df.loc[i - 1, \"up_left_y\"] + bottom_y_distances[i - 1]\n", " ],\n", " }\n", " ),\n", " ],\n", " ignore_index=True,\n", " )\n", " print(\"last 100 df:\", df[\"up_left_y\"].values[-100:])\n", " print(\"length:\", len(df))\n", "\n", " for i, row in df.iterrows():\n", " y = row[\"up_left_y\"]\n", " left_intersection = get_intersection(\n", " (points_df.loc[i, \"bottom_left_x\"], points_df.loc[i, \"bottom_left_y\"]),\n", " (points_df.loc[i, \"up_left_x\"], points_df.loc[i, \"up_left_y\"]),\n", " (0, y),\n", " (1000, y),\n", " )\n", " right_intersection = get_intersection(\n", " (points_df.loc[i, \"bottom_right_x\"], points_df.loc[i, \"bottom_right_y\"]),\n", " (points_df.loc[i, \"up_right_x\"], points_df.loc[i, \"up_right_y\"]),\n", " (0, y),\n", " (1000, y),\n", " )\n", " if left_intersection is not None and right_intersection is not None:\n", " points_df.loc[i, \"up_left_x\"] = left_intersection[0]\n", " points_df.loc[i, \"up_left_y\"] = left_intersection[1]\n", " points_df.loc[i, \"up_right_x\"] = right_intersection[0]\n", " points_df.loc[i, \"up_right_y\"] = right_intersection[1]\n", " return points_df" ] }, { "cell_type": "markdown", "id": "2431a680", "metadata": {}, "source": [ "Main Algorithm" ] }, { "cell_type": "code", "execution_count": 106, "id": "19188541", "metadata": {}, "outputs": [], "source": [ "def postprocessing(points_df, heigth):\n", " # initial_points = points_df.iloc[0]\n", " bottom_disappeared = False\n", " frame = 10\n", " # compute the relative positions of points in the first frame\n", " # left_relative_position = (points_df.iloc[frame][\"up_left_x\"]-points_df.iloc[frame][\"bottom_left_x\"], points_df.iloc[frame][\"up_left_y\"]-points_df.iloc[frame][\"bottom_left_y\"])\n", " # right_relative_position = (points_df.iloc[frame][\"up_right_x\"]-points_df.iloc[frame][\"bottom_right_x\"], points_df.iloc[frame][\"up_right_y\"]-points_df.iloc[frame][\"bottom_right_y\"])\n", "\n", " df_copy = points_df.copy()\n", " bottom_y_distances = np.diff(df_copy[\"bottom_left_y\"].values)\n", "\n", " for i in range(1, len(points_df)):\n", " bl_prev = (\n", " df_copy.iloc[i - 1][\"bottom_left_x\"],\n", " df_copy.iloc[i - 1][\"bottom_left_y\"],\n", " )\n", " br_prev = (\n", " df_copy.iloc[i - 1][\"bottom_right_x\"],\n", " df_copy.iloc[i - 1][\"bottom_right_y\"],\n", " )\n", " tl_prev = (df_copy.iloc[i - 1][\"up_left_x\"], df_copy.iloc[i - 1][\"up_left_y\"])\n", " tr_prev = (df_copy.iloc[i - 1][\"up_right_x\"], df_copy.iloc[i - 1][\"up_right_y\"])\n", " bl = (points_df.iloc[i][\"bottom_left_x\"], points_df.iloc[i][\"bottom_left_y\"])\n", " br = (points_df.iloc[i][\"bottom_right_x\"], points_df.iloc[i][\"bottom_right_y\"])\n", " tr = (points_df.iloc[i][\"up_right_x\"], points_df.iloc[i][\"up_right_y\"])\n", " tl = (points_df.iloc[i][\"up_left_x\"], points_df.iloc[i][\"up_left_y\"])\n", "\n", " if not bottom_disappeared:\n", " # compute relative positions\n", " window_size = 7\n", " if i < window_size:\n", " left_relative_position = (\n", " tl_prev[0] - bl_prev[0],\n", " tl_prev[1] - bl_prev[1],\n", " )\n", " right_relative_position = (\n", " tr_prev[0] - br_prev[0],\n", " tr_prev[1] - br_prev[1],\n", " )\n", " else:\n", " left_relative_positions = [\n", " (\n", " df_copy.iloc[j][\"up_left_x\"] - df_copy.iloc[j][\"bottom_left_x\"],\n", " df_copy.iloc[j][\"up_left_y\"] - df_copy.iloc[j][\"bottom_left_y\"],\n", " )\n", " for j in range(i - window_size + 1, i + 1)\n", " ]\n", " right_relative_positions = [\n", " (\n", " df_copy.iloc[j][\"up_right_x\"]\n", " - df_copy.iloc[j][\"bottom_right_x\"],\n", " df_copy.iloc[j][\"up_right_y\"]\n", " - df_copy.iloc[j][\"bottom_right_y\"],\n", " )\n", " for j in range(i - window_size + 1, i + 1)\n", " ]\n", "\n", " # # mean\n", " # left_relative_position = (sum(pos[0] for pos in left_relative_positions) / window_size,\n", " # sum(pos[1] for pos in left_relative_positions) / window_size)\n", " # right_relative_position = (sum(pos[0] for pos in right_relative_positions) / window_size,\n", " # sum(pos[1] for pos in right_relative_positions) / window_size)\n", "\n", " # lower\n", " left_relative_position = sorted(\n", " left_relative_positions, key=lambda pos: pos[1], reverse=True\n", " )[1]\n", " right_relative_position = sorted(\n", " right_relative_positions, key=lambda pos: pos[1], reverse=True\n", " )[1]\n", " # compute the new position of the bottom points in the current frame () if needed\n", " bl_new, br_new = is_disappeared(bl_prev, br_prev, bl, br, tr, tl, heigth)\n", "\n", " if bl_new is None and br_new is None:\n", " bottom_disappeared = True\n", " index_disappeared = i\n", " print(f\"Bottom points disappeared at frame {i}.\")\n", " else: # consider correct the bottom points\n", " tr_mid = (\n", " br_new[0] + right_relative_position[0],\n", " br_new[1] + right_relative_position[1],\n", " )\n", " tl_mid = (\n", " bl_new[0] + left_relative_position[0],\n", " bl_new[1] + left_relative_position[1],\n", " )\n", " # Calculate the intersection point\n", " tr_new = get_intersection(tr_mid, tl_mid, br_new, tr)\n", " tl_new = get_intersection(tl_mid, tr_mid, bl_new, tl)\n", "\n", " if bottom_disappeared: # consider correct the top poits\n", " bl_new = (\n", " tl[0] - left_relative_position[0],\n", " tl[1] - left_relative_position[1],\n", " )\n", " br_new = (\n", " tr[0] - right_relative_position[0],\n", " tr[1] - right_relative_position[1],\n", " )\n", " tr_new = tr\n", " tl_new = tl\n", "\n", " # Update the DataFrame with the new points\n", " points_df.at[i, \"bottom_left_x\"] = bl_new[0]\n", " points_df.at[i, \"bottom_left_y\"] = bl_new[1]\n", " points_df.at[i, \"bottom_right_x\"] = br_new[0]\n", " points_df.at[i, \"bottom_right_y\"] = br_new[1]\n", " points_df.at[i, \"up_left_x\"] = tl_new[0]\n", " points_df.at[i, \"up_left_y\"] = tl_new[1]\n", " points_df.at[i, \"up_right_x\"] = tr_new[0]\n", " points_df.at[i, \"up_right_y\"] = tr_new[1]\n", " df = postprocessing_upper(points_df, bottom_y_distances)\n", " if bottom_disappeared:\n", " for i in range(index_disappeared, len(points_df)):\n", " # compute the bottom points in the rows that are changed between df and points_df with relative positions\n", " bl_new = (\n", " df.iloc[i][\"up_left_x\"] - left_relative_position[0],\n", " df.iloc[i][\"up_left_y\"] - left_relative_position[1],\n", " )\n", " br_new = (\n", " df.iloc[i][\"up_right_x\"] - right_relative_position[0],\n", " df.iloc[i][\"up_right_y\"] - right_relative_position[1],\n", " )\n", "\n", " df.at[i, \"bottom_left_x\"] = bl_new[0]\n", " df.at[i, \"bottom_left_y\"] = bl_new[1]\n", " df.at[i, \"bottom_right_x\"] = br_new[0]\n", " df.at[i, \"bottom_right_y\"] = br_new[1]\n", "\n", " return df" ] }, { "cell_type": "markdown", "id": "f105fb56", "metadata": {}, "source": [ "Auxiliary functions" ] }, { "cell_type": "code", "execution_count": 107, "id": "81afef62", "metadata": {}, "outputs": [], "source": [ "def lines_from_points(df):\n", " lines = []\n", " for i in range(len(df)):\n", " bottom = [\n", " df.iloc[i][\"bottom_right_x\"],\n", " df.iloc[i][\"bottom_right_y\"],\n", " df.iloc[i][\"bottom_left_x\"],\n", " df.iloc[i][\"bottom_left_y\"],\n", " ]\n", " top = [\n", " df.iloc[i][\"up_right_x\"],\n", " df.iloc[i][\"up_right_y\"],\n", " df.iloc[i][\"up_left_x\"],\n", " df.iloc[i][\"up_left_y\"],\n", " ]\n", " left = [\n", " df.iloc[i][\"bottom_left_x\"],\n", " df.iloc[i][\"bottom_left_y\"],\n", " df.iloc[i][\"up_left_x\"],\n", " df.iloc[i][\"up_left_y\"],\n", " ]\n", " right = [\n", " df.iloc[i][\"bottom_right_x\"],\n", " df.iloc[i][\"bottom_right_y\"],\n", " df.iloc[i][\"up_right_x\"],\n", " df.iloc[i][\"up_right_y\"],\n", " ]\n", "\n", " # Append the line to the list\n", " lines.append([bottom, top, left, right])\n", " return lines" ] }, { "cell_type": "code", "execution_count": 108, "id": "f9e9b300", "metadata": {}, "outputs": [], "source": [ "\"\"\"Disegna la linea sul frame\"\"\"\n", "\n", "\n", "def write_line_on_frame(frame, line):\n", " # Create a copy of the original frame to draw the first line\n", " modified_frame = np.copy(frame)\n", "\n", " # Extract the first line's rho and theta\n", " if line is not None:\n", " x1, y1, x2, y2 = line\n", "\n", " # Draw the first line on the frame\n", " cv2.line(modified_frame, (int(x1), int(y1)), (int(x2), int(y2)), (0, 255, 0), 2)\n", "\n", " # return the modified frame\n", " return modified_frame" ] }, { "cell_type": "code", "execution_count": 109, "id": "b4088b1f", "metadata": {}, "outputs": [], "source": [ "def write_lines_on_frame(frame, lines):\n", " for i in range(len(lines)):\n", " # print('linea', i, ':', lines[i])\n", " frame = write_line_on_frame(frame, lines[i])\n", " return frame" ] }, { "cell_type": "markdown", "id": "16eea54d", "metadata": {}, "source": [ "Generate the video" ] }, { "cell_type": "code", "execution_count": 110, "id": "7563fa98", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Bottom points disappeared at frame 110.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "C:\\Users\\miche\\AppData\\Local\\Temp\\ipykernel_4716\\2827118193.py:72: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise an error in a future version of pandas. Value '1244.9496606334842' has dtype incompatible with int64, please explicitly cast to a compatible dtype first.\n", " points_df.at[i, \"up_left_x\"] = tl_new[0]\n", "C:\\Users\\miche\\AppData\\Local\\Temp\\ipykernel_4716\\2827118193.py:73: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise an error in a future version of pandas. Value '445.01300904977376' has dtype incompatible with int64, please explicitly cast to a compatible dtype first.\n", " points_df.at[i, \"up_left_y\"] = tl_new[1]\n", "C:\\Users\\miche\\AppData\\Local\\Temp\\ipykernel_4716\\2827118193.py:74: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise an error in a future version of pandas. Value '1536.9095394736842' has dtype incompatible with int64, please explicitly cast to a compatible dtype first.\n", " points_df.at[i, \"up_right_x\"] = tr_new[0]\n", "C:\\Users\\miche\\AppData\\Local\\Temp\\ipykernel_4716\\2827118193.py:75: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise an error in a future version of pandas. Value '449.012459444845' has dtype incompatible with int64, please explicitly cast to a compatible dtype first.\n", " points_df.at[i, \"up_right_y\"] = tr_new[1]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Mean distance: -1.1017699115044248\n", "Std distance: 39.52051537789164\n", "last 100 df: [459. 459. 459. 459. 460. 460. 460. 462. 460. 460. 460. 461. 461. 459.\n", " 462. 464. 462. 464. 463. 467. 471. 474. 477. 482. 485. 486. 486. 492.\n", " 493. 496. 495. 493. 494. 494. 501. 504. 504. 504. 500. 504. 505. 505.\n", " 508. 505. 504. 506. 507. 508. 508. 508. 509. 508. 510. 510. 510. 511.\n", " 506. 507. 509. 509. 509. 510. 510. 510. 512. 515. 513. 513. 513. 513.\n", " 513. 513. 513. 513. 513. 513. 513. 513. 513. 513. 513. 513. 513. 513.\n", " 513. 513. 513. 513. 513. 513. 513. 513. 513. 513. 513. 513. 513. 513.\n", " 513. 513.]\n", "length: 227\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "C:\\Users\\miche\\AppData\\Local\\Temp\\ipykernel_4716\\2827118193.py:85: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise an error in a future version of pandas. Value '1041.3792544570501' has dtype incompatible with int64, please explicitly cast to a compatible dtype first.\n", " df.at[i, \"bottom_right_x\"] = br_new[0]\n", "C:\\Users\\miche\\AppData\\Local\\Temp\\ipykernel_4716\\2827118193.py:83: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise an error in a future version of pandas. Value '446.375' has dtype incompatible with int64, please explicitly cast to a compatible dtype first.\n", " df.at[i, \"bottom_left_x\"] = bl_new[0]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Adjusted video saved to C:\\Users\\miche\\OneDrive\\Documenti\\GitHub\\bowling-analysis\\data\\recording_3\\test_video\\Lane_1.mp4\n" ] } ], "source": [ "# Reset the video to the beginning\n", "cap.set(cv2.CAP_PROP_POS_FRAMES, 0)\n", "\n", "# Define the codec and create a VideoWriter object to save the modified frames\n", "output_path = (\n", " project_root / \"data\" / f\"recording_{video_number}\" / \"test_video\" / \"Lane_1.mp4\"\n", ")\n", "fourcc = cv2.VideoWriter_fourcc(*\"mp4v\") # Use 'mp4v' codec for MP4 format\n", "fps = int(cap.get(cv2.CAP_PROP_FPS))\n", "frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))\n", "frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))\n", "out = cv2.VideoWriter(str(output_path), fourcc, fps, (frame_width, frame_height))\n", "\n", "# Process the lines\n", "processed_points_df = postprocessing(points_df, frame_height)\n", "lines = lines_from_points(processed_points_df)\n", "# print('Processed lines:', lines)\n", "\n", "\n", "# Loop through each frame in the video\n", "frame_index = 0\n", "while frame_index < len(lines):\n", " ret, video_frame = cap.read()\n", " if not ret:\n", " print(\"End of video or failed to read the frame at iteration\", frame_index)\n", " break\n", " # print(f\"Processing frame {frame_index}\")\n", "\n", " # draw the lines on the frame\n", " modified_frame = write_lines_on_frame(video_frame, lines[frame_index])\n", "\n", " # Write the modified frame to the output video\n", " out.write(modified_frame)\n", "\n", " # Increment the frame index\n", " frame_index += 1\n", "\n", "# Release the video capture and writer objects\n", "# cap.release()\n", "out.release()\n", "\n", "print(f\"Adjusted video saved to {output_path}\")" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.9" } }, "nbformat": 4, "nbformat_minor": 5 }