diff --git a/affine/CMakeLists.txt b/affine/CMakeLists.txt new file mode 100755 index 0000000..8ff4a31 --- /dev/null +++ b/affine/CMakeLists.txt @@ -0,0 +1,30 @@ +# requires minimum cmake version +cmake_minimum_required(VERSION 3.5) + +# produce the cmake var PROJECT_NAME +project(shape_position) + +if(NOT CMAKE_BUILD_TYPE) +set_property(CACHE CMAKE_BUILD_TYPE PROPERTY VALUE "Release") +endif() + +#include(GNUInstallDirs) + +# mandatory use of these packages +find_package(YCM REQUIRED) +find_package(OpenCV REQUIRED) +find_package(YARP COMPONENTS OS sig math dev REQUIRED) +find_package(event-driven REQUIRED) + +add_executable(${PROJECT_NAME} main.cpp eros.h affine.h) + +target_link_libraries(${PROJECT_NAME} PRIVATE YARP::YARP_OS + YARP::YARP_init + YARP::YARP_sig + YARP::YARP_dev + YARP::YARP_math + ev::event-driven + ${OpenCV_LIBRARIES}) + +install(TARGETS ${PROJECT_NAME} DESTINATION ${CMAKE_INSTALL_BINDIR}) + diff --git a/affine/affine.h b/affine/affine.h new file mode 100755 index 0000000..8170576 --- /dev/null +++ b/affine/affine.h @@ -0,0 +1,547 @@ +#pragma once + +#include + +using namespace yarp::os; + +class affineTransforms +{ +public: + + enum cam_param_name{w,h,cx,cy,fx,fy}; + + typedef struct affine_struct + { + cv::Mat A; + cv::Mat warped_img; + double score; + cv::Mat rmp; + cv::Mat rmsp; + int axis{0}; + double delta{0.0}; + + } affine_bundle; + + std::array affine_info; + std::vector affines_vector; + std::vector scores_vector; + std::array state; + + cv::Rect2d roi_around_shape, square; + cv::Mat initial_template, roi_template, roi_template_64f, roi_resized, mexican_template, mexican_resized, mexican_template_64f; + cv::Mat eros_filtered, eros_tracked, eros_tracked_64f, eros_resized; + cv::Mat rot_scaled_tr_template; + double translation, angle, pscale, nscale; + cv::Point2d initial_position, new_position; + cv::Point2d new_center; + int blur{7}; + int gaussian_blur_eros{5}; + double template_scale{0.4}; //0.4, 0.33 + + cv::Size proc_size{cv::Size(50, 50)}; + cv::Rect proc_roi, o_proc_roi; + cv::Mat prmx, prmy, nrmx, nrmy; + + std::array cam; + +public: + + void init(double trans, double rot, double ps, double ns){ + + roi_resized = cv::Mat(proc_size, CV_64F); + eros_resized = cv::Mat(proc_size, CV_64F); + mexican_resized = cv::Mat(proc_size, CV_64F); + o_proc_roi = proc_roi = cv::Rect(cv::Point(0, 0), proc_size); + + for(auto &affine : affine_info) { + affine.A = cv::Mat::zeros(2,3, CV_64F); + affine.warped_img = cv::Mat::zeros(proc_size, CV_64F); + } + + prmx = cv::Mat::zeros(proc_size, CV_32F); + prmy = cv::Mat::zeros(proc_size, CV_32F); + nrmx = cv::Mat::zeros(proc_size, CV_32F); + nrmy = cv::Mat::zeros(proc_size, CV_32F); + + this->translation = trans; + this->angle = rot; + this->pscale = ps; + this->nscale = ns; + } + + void initState(){ + state[0]=0; state[1]=0; state[2]=0; state[3]=1; + } + + void createAffines(double translation, cv::Point2f center, double angle, double pscale, double nscale){ + + affine_info[0].A.at(0,0) = 1; + affine_info[0].A.at(0,2) = translation; + affine_info[0].A.at(1,1) = 1; + affine_info[0].A.at(1,2) = 0; + + affine_info[1].A.at(0,0) = 1; + affine_info[1].A.at(0,2) = -translation; + affine_info[1].A.at(1,1) = 1; + affine_info[1].A.at(1,2) = 0; + + affine_info[2].A.at(0,0) = 1; + affine_info[2].A.at(0,2) = 0; + affine_info[2].A.at(1,1) = 1; + affine_info[2].A.at(1,2) = translation; + + affine_info[3].A.at(0,0) = 1; + affine_info[3].A.at(0,2) = 0; + affine_info[3].A.at(1,1) = 1; + affine_info[3].A.at(1,2) = -translation; + + affine_info[4].A = cv::getRotationMatrix2D(center, angle, 1); + affine_info[5].A = cv::getRotationMatrix2D(center, -angle, 1); + + affine_info[6].A = cv::getRotationMatrix2D(center, 0, pscale); + affine_info[7].A = cv::getRotationMatrix2D(center, 0, nscale); + + affine_info[8].A.at(0,0) = 1; + affine_info[8].A.at(1,1) = 1; + } + + void updateAffines(){ + + new_center = cv::Point2d(roi_around_shape.width/2, roi_around_shape.height/2); + + affine_info[4].A = cv::getRotationMatrix2D(new_center, angle, 1); + affine_info[5].A = cv::getRotationMatrix2D(new_center, -angle, 1); + + affine_info[6].A = cv::getRotationMatrix2D(new_center, 0, pscale); + affine_info[7].A = cv::getRotationMatrix2D(new_center, 0, nscale); + } + + void setROI(int buffer = 0){ + + roi_resized = 0; + static cv::Rect2d full_roi = cv::Rect2d(cv::Point(0, 0), rot_scaled_tr_template.size()); + + roi_around_shape = cv::boundingRect(rot_scaled_tr_template); + + if(roi_around_shape.width == 0) { + roi_around_shape = full_roi; + return; + } + roi_around_shape.x -= buffer; + roi_around_shape.y -= buffer; + roi_around_shape.width += buffer * 2; + roi_around_shape.height += buffer * 2; + + roi_around_shape = roi_around_shape & full_roi; + + roi_template = rot_scaled_tr_template(roi_around_shape); + roi_template.convertTo(roi_template_64f, CV_64F); // is 6 type CV_64FC1 + + cv::resize(roi_template_64f, roi_resized(proc_roi), proc_roi.size(), 0, 0, cv::INTER_LINEAR); + make_template(roi_resized, mexican_resized); + } + + cv::Mat updateTrMat(double translation_x, double translation_y){ + + // the state is an affine + cv::Mat trMat = cv::Mat::zeros(2,3, CV_64F); + + trMat.at(0,0) = 1; trMat.at(0,1) = 0; trMat.at(0,2) = translation_x; + trMat.at(1,0) = 0; trMat.at(1,1) = 1; trMat.at(1,2) = translation_y; + + return trMat; + } + + void createStaticTemplate(const cv::Size &res, double scaling_percentage){ + + initial_template = cv::Mat::zeros(res.height, res.width, CV_8UC1); + double square_width = res.height*(scaling_percentage/100); + double square_height = res.height*(scaling_percentage/100); + square = cv::Rect2d((res.width-square_width)/2,(res.height-square_height)/2, square_width, square_height); + cv::rectangle(initial_template, square, cv::Scalar(255),1,8,0); + + initial_position.x = square.x + square.width/2; + initial_position.y = square.y + square.height/2; + + } + + void squareTemplate (const cv::Size &res){ + initial_template = cv::Mat::zeros(res.height, res.width, CV_8UC1); + square = cv::Rect(306,263, 87, 93); + cv::rectangle(initial_template, square, cv::Scalar(255),1,8,0); + + initial_position.x = square.x + square.width/2; + initial_position.y = square.y + square.height/2; + } + + void starTemplate(const cv::Size &res){ + + initial_template = cv::Mat::zeros(res.height, res.width, CV_8UC1); + + cv::line(initial_template, cv::Point(327,172), cv::Point(336,204), cv::Scalar(255),1,8,0); + cv::line(initial_template, cv::Point(336,204), cv::Point(367,208), cv::Scalar(255),1,8,0); + cv::line(initial_template, cv::Point(367,208), cv::Point(341,226), cv::Scalar(255),1,8,0); + cv::line(initial_template, cv::Point(341,226), cv::Point(348,258), cv::Scalar(255),1,8,0); + cv::line(initial_template, cv::Point(348,258), cv::Point(324,237), cv::Scalar(255),1,8,0); + cv::line(initial_template, cv::Point(324,237), cv::Point(296,256), cv::Scalar(255),1,8,0); + cv::line(initial_template, cv::Point(296,256), cv::Point(308,225), cv::Scalar(255),1,8,0); + cv::line(initial_template, cv::Point(308,225), cv::Point(283,203), cv::Scalar(255),1,8,0); + cv::line(initial_template, cv::Point(283,203), cv::Point(316,202), cv::Scalar(255),1,8,0); + cv::line(initial_template, cv::Point(316,202), cv::Point(327,172), cv::Scalar(255),1,8,0); + + initial_position.x = 325; + initial_position.y = 219; + } + + void loadTemplate(const cv::Size &res, std::string filename){ + + cv::Mat shape_image = cv::imread(filename, 0); + + cv::resize(shape_image, shape_image, cv::Size(template_scale*shape_image.cols, template_scale*shape_image.rows), 0, 0, cv::INTER_CUBIC); + + // yInfo()<(0, 2) += (state[0] - initial_position.x); + // rotMatfunc.at(1, 2) += (state[1] - initial_position.y); + // cv::warpAffine(initial_template, rot_scaled_tr_template, rotMatfunc, rot_scaled_tr_template.size()); + // new_position = cv::Point2d(initial_position.x+state[0],initial_position.y+state[1]); + + // } + + void createDynamicTemplate(){ + + // cv::Mat rot_scaled_template; + + // rot_scaled_tr_template = 0; + + cv::Mat rotMatfunc = getRotationMatrix2D(initial_position, state[2], state[3]); + // cv::warpAffine(initial_template, rot_scaled_template, rotMatfunc, rot_scaled_template.size()); + // cv::Mat trMat = updateTrMat(state[0], state[1]); + rotMatfunc.at(0,2) += (state[0]); + rotMatfunc.at(1,2) += (state[1]); + cv::warpAffine(initial_template, rot_scaled_tr_template, rotMatfunc, initial_template.size()); + // cv::warpAffine(rot_scaled_template, rot_scaled_tr_template, rotMatfunc, rot_scaled_template.size()); + new_position = cv::Point2d(initial_position.x+state[0],initial_position.y+state[1]); + + } + + void make_template(const cv::Mat &input, cv::Mat &output) { + + static cv::Mat canny_img, f, pos_hat, neg_hat; + static cv::Size pblur(blur, blur); + static cv::Size nblur(2*blur-1, 2*blur-1); + static double minval, maxval; + + cv::GaussianBlur(input, pos_hat, pblur, 0); + cv::GaussianBlur(input, neg_hat, nblur, 0); + // cv::GaussianBlur(roi_template_64f, pos_hat, pblur, 0); + // cv::GaussianBlur(roi_template_64f, neg_hat, nblur, 0); + output = pos_hat - neg_hat; + + cv::minMaxLoc(output, &minval, &maxval); + double scale_factor = 1.0 / (2 * std::max(fabs(minval), fabs(maxval))); + output *= scale_factor; + } + + void createWarpings(){ + + mexican_template.convertTo(mexican_template_64f, CV_64F); // is 6 type CV_64FC1 + + for (int affine = 0; affine < affine_info.size(); affine++) { + cv::warpAffine(mexican_template_64f, affine_info[affine].warped_img, affine_info[affine].A, mexican_template_64f.size()); + } + } + + void createMapWarpings(){ + + // std::vector affine_matrices; + // std::vector couple_matrix; + + // cv::Mat concat_affines; + mexican_resized.convertTo(mexican_template_64f, CV_64F); // is 6 type CV_64FC1 + + for (int affine = 0; affine < affine_info.size()-1; affine++) { + cv::remap(mexican_template_64f, affine_info[affine].warped_img, affine_info[affine].rmp, affine_info[affine].rmsp, cv::INTER_LINEAR); + // affine_matrices.push_back(affine_info[affine].warped_img); + // cv::imshow("affine remap" + std::to_string(affine), affine_info[affine].warped_img); + } + + affine_info[8].warped_img = mexican_template_64f; + // for (int i=0; i +#include +#include +#include +using namespace yarp::os; + +#include "eros.h" +#include "affine.h" + +class affineTracking: public yarp::os::RFModule +{ +private: + + cv::Size img_size; + double period{0.01}; + double eros_k, eros_d; + double tau_latency{0}; + double dT{0}; + double recording_duration, elapsed_time{0}; + float translation{2}, angle{0.1}, pscale{1.0001}, nscale{0.9999}; + bool run{false}; + double dt_warpings{0}, dt_comparison{0}, dt_eros{0}, toc_count{0}; + std::string filename; + std::deque< std::array > data_to_save; + std::ofstream fs; + std::vector gt_values; + bool gt_sending{false}; + int gt_index{0}; + + struct fake_latency{ + std::array state; + double tstamp; + }; + + std::deque fakeLat_queue; + + EROSfromYARP eros_handler; + affineTransforms affine_handler; + std::thread computation_thread; + +public: + + std::vector readFromCsv(std::string fname){ + + std::vector> content; + std::vector times, gt_values; + std::string line, word, ts, gt; + + std::fstream file (fname, std::ios::in); + if(file.is_open()) + { + while(std::getline(file, line)) + { + std::stringstream str(line); + + getline(str, ts, ' '); + times.push_back(std::stod(ts)); + getline(str, gt, '\n'); + gt_values.push_back(std::stod(gt)); + + } + } + else + std::cout<<"Could not open the file\n"; + + // for(int i=0;i recording_duration){ + yInfo()<<"recording duration reached"; + return false; + } + + // std::cout<dt_warpings = toc_warpings - tic_warpings; + // this->dt_comparison = toc_comparison - tic_comparison; + // this->dt_eros = toc_eros - tic_eros; + // this->toc_count++; + + // yInfo()<<"dt"< +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace yarp::os; +using namespace yarp::sig; +using namespace std; + +class PID +{ + double Kp,Ki; + double integral; + +public: + // constructor + PID() : Kp(0.0), Ki(0.0), integral(0.0) { } + + // helper function to set up sample time and gains + void set(const double Kp, const double Ki) + { + this->Kp=Kp; + this->Ki=Ki; + } + + // compute the control command + double command(const double reference, const double feedback, const double Ts) + { + // the actual error between reference and feedback + double error=reference-feedback; + + // accumulate the error + integral+=error*Ts; + + // compute the PID output + return (Kp*error+Ki*integral); + } + + void reset() + { + integral = 0; + } +}; + +class headControl { + +private: + + yarp::dev::PolyDriver vel_driver, pos_driver; + yarp::dev::IEncoders *ienc; + yarp::dev::IControlMode *imod; + yarp::dev::IVelocityControl *ivel; + yarp::dev::IPositionControl *ipos; + yarp::dev::IControlLimits *ilim; + + int nAxes; + std::vector controllers; + std::vector velocity; + std::vector encs; + + int u_fixation; + int v_fixation; + +protected: + +public: + + headControl() : nAxes(6), u_fixation(0), v_fixation(0) {} + + bool initVelControl(int height, int width) + { + yarp::os::Property option; + option.put("device","remote_controlboard"); + option.put("remote","/icub/head"); + option.put("local","/vel_controller"); + + if (!vel_driver.open(option)) + { + yError()<<"Unable to open the device vel_driver"; + return false; + } + + // open the views + + if(!vel_driver.view(ienc)) { + yError() << "Driver does not implement encoder mode"; + return false; + } + if(!vel_driver.view(imod)) { + yError() << "Driver does not implement control mode"; + return false; + } + if(!vel_driver.view(ivel)) { + yError() << "Driver does not implement velocity mode"; + return false; + } + + // retrieve number of axes + int readAxes; + ienc->getAxes(&readAxes); + if(readAxes != nAxes) { + yError() << "Incorrect number of axes" << readAxes << nAxes; + return false; + } + + velocity.resize(nAxes, 0.0); + encs.resize(nAxes); + controllers.resize(nAxes); + for(int i = 0; i < nAxes; i++) + controllers[i] = new PID; + + // set up our controllers + controllers[0]->set(2.0, 0.0); //neck pitch + controllers[1]->set(0.0, 0.0); //neck roll + controllers[2]->set(2.0, 0.0); //neck yaw + + u_fixation = width / 2; + v_fixation = height / 2; + + //set velocity control mode + return setVelocityControl(); + + } + + bool initPosControl(){ + yarp::os::Property option; + option.put("device", "remote_controlboard"); + option.put("remote", "/icub/head"); + option.put("local", "/pos_controller"); + + pos_driver.open(option); + pos_driver.view(ipos); + pos_driver.view(imod); + pos_driver.view(ilim); + + return true; + } + + bool setVelocityControl() + { + int naxes; + ivel->getAxes(&naxes); + std::vector modes(naxes, VOCAB_CM_VELOCITY); + + imod->setControlModes(modes.data()); + + return true; + } + + void resetRobotHome(){ + + int naxes; + ipos->getAxes(&naxes); + std::vector modes(naxes, VOCAB_CM_POSITION); + std::vector vels(naxes, 10.); + std::vector accs(naxes, std::numeric_limits::max()); + std::vector poss(naxes, 0.); + poss[2]=0; + poss[0]=-23; + + imod->setControlModes(modes.data()); + ipos->setRefSpeeds(vels.data()); + ipos->setRefAccelerations(accs.data()); + ipos->positionMove(poss.data()); + + auto done = false; + while(!done) { + yarp::os::Time::delay(1.); + ipos->checkMotionDone(&done); + } + + setVelocityControl(); + } + + void scroll_yaw(){ + int naxes; + ipos->getAxes(&naxes); + std::vector modes(naxes, VOCAB_CM_POSITION); + std::vector vels(naxes, 20.); + std::vector accs(naxes, std::numeric_limits::max()); + std::vector poss(naxes, 0.); + poss[2]=-10; + poss[0]=5.977; + + imod->setControlModes(modes.data()); + ipos->setRefSpeeds(vels.data()); + ipos->setRefAccelerations(accs.data()); + ipos->positionMove(poss.data()); + + auto done = false; + while(!done) { + yarp::os::Time::delay(1.); + ipos->checkMotionDone(&done); + } + } + + void controlMono(int u, int v, double dt) + { + + double neck_tilt=controllers[0]->command(v_fixation,v, dt); // neck pitch + double neck_pan=controllers[2]->command(u_fixation,u, dt); // neck yaw + + // send commands to the robot head + velocity[0]=neck_tilt; // neck pitch + velocity[1]=0.0; // neck roll + velocity[2]=neck_pan; // neck yaw + velocity[3]=0.0; // eyes tilt + velocity[4]=0.0; // eyes vers + velocity[5]=0.0; // eyes verg + +// yInfo()<<"vel: "<velocityMove(0, neck_tilt); + //ivel->velocityMove(2, neck_pan); + ivel->velocityMove(velocity.data()); + +// double vel_tilt, vel_pan; +// ivel->getRefVelocity(0,&vel_tilt); +// ivel->getRefVelocity(2,&vel_pan); +// yInfo()<<"current vel"<getEncoders(encs.data()); + + double eyes_pan=-controllers[4]->command(u_fixation, ul, dt); + double eyes_tilt=controllers[3]->command(v_fixation, vl, dt); + if(encs[3] < -6 && (v_fixation - vl) < 0) { + eyes_tilt = 0.0; + controllers[3]->reset(); + } + if(encs[3] > 20 && (v_fixation - vl) > 0) { + eyes_tilt = 0.0; + controllers[3]->reset(); + } + double eyes_ver=-controllers[5]->command(0,ul-ur, dt); + + // feed-forward correction to reduce + // interplay between vergence and pan + eyes_pan-=eyes_ver/2.0; + + double neck_tilt=-controllers[0]->command(0.0,encs[3], dt); + double neck_pan=controllers[2]->command(0.0,encs[4], dt); + + // send commands to the robot head + velocity[0]=neck_tilt; // neck pitch + velocity[1]=0.0; // neck roll + velocity[2]=neck_pan; // neck yaw + velocity[3]=eyes_tilt; // eyes tilt + velocity[4]=eyes_pan; // eyes pan + velocity[5]=eyes_ver; // eyes vergence + ivel->velocityMove(velocity.data()); + } + + void controlReset() + { + for(int i = 0; i < nAxes; i++) { + controllers[i]->reset(); + velocity[i] = 0; + } + ivel->velocityMove(velocity.data()); + } + + double getJointPos(int joint_number){ + ienc->getEncoders(encs.data()); + return encs[joint_number]; + } + + void getJointLimits(int joint_num, double* joint_min, double* joint_max){ + + double min, max; + ilim->getLimits(joint_num, &min, &max); + + *joint_min = min; + *joint_max = max; + } + + void closeToLimit(int joint_number){ + + double joint_pos_min, joint_pos_max, current_joint; + ienc->getEncoders(encs.data()); + if (joint_number == 0){ + joint_pos_min=-27; joint_pos_max=20; + current_joint = encs[0]; + } + if (joint_number == 2){ + joint_pos_min=-44; joint_pos_max=44; + current_joint = encs[2]; + } + +// getJointLimits(joint_number, &joint_pos_min, &joint_pos_max); + + double error_min = current_joint - joint_pos_max; + double error_max = current_joint - joint_pos_min; + + double smallTh = 3; + + if (error_min < smallTh || error_max < smallTh){ + resetRobotHome(); + yInfo()<<"joints resetted"; + } + } + + + +}; +