/** * This file is part of ORB-SLAM3 * * Copyright (C) 2017-2020 Carlos Campos, Richard Elvira, Juan J. Gómez Rodríguez, José M.M. Montiel and Juan D. Tardós, University of Zaragoza. * Copyright (C) 2014-2016 Raúl Mur-Artal, José M.M. Montiel and Juan D. Tardós, University of Zaragoza. * * ORB-SLAM3 is free software: you can redistribute it and/or modify it under the terms of the GNU General Public * License as published by the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * ORB-SLAM3 is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even * the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License along with ORB-SLAM3. * If not, see . */ #include "MapPoint.h" #include "ORBmatcher.h" #include namespace ORB_SLAM3 { long unsigned int MapPoint::nNextId=0; mutex MapPoint::mGlobalMutex; MapPoint::MapPoint(): mnFirstKFid(0), mnFirstFrame(0), nObs(0), mnTrackReferenceForFrame(0), mnLastFrameSeen(0), mnBALocalForKF(0), mnFuseCandidateForKF(0), mnLoopPointForKF(0), mnCorrectedByKF(0), mnCorrectedReference(0), mnBAGlobalForKF(0), mnVisible(1), mnFound(1), mbBad(false), mpReplaced(static_cast(NULL)) { mpReplaced = static_cast(NULL); } MapPoint::MapPoint(const cv::Mat &Pos, KeyFrame *pRefKF, Map* pMap): mnFirstKFid(pRefKF->mnId), mnFirstFrame(pRefKF->mnFrameId), nObs(0), mnTrackReferenceForFrame(0), mnLastFrameSeen(0), mnBALocalForKF(0), mnFuseCandidateForKF(0), mnLoopPointForKF(0), mnCorrectedByKF(0), mnCorrectedReference(0), mnBAGlobalForKF(0), mpRefKF(pRefKF), mnVisible(1), mnFound(1), mbBad(false), mpReplaced(static_cast(NULL)), mfMinDistance(0), mfMaxDistance(0), mpMap(pMap), mnOriginMapId(pMap->GetId()) { Pos.copyTo(mWorldPos); mWorldPosx = cv::Matx31f(Pos.at(0), Pos.at(1), Pos.at(2)); mNormalVector = cv::Mat::zeros(3,1,CV_32F); mNormalVectorx = cv::Matx31f::zeros(); mbTrackInViewR = false; mbTrackInView = false; // MapPoints can be created from Tracking and Local Mapping. This mutex avoid conflicts with id. unique_lock lock(mpMap->mMutexPointCreation); mnId=nNextId++; } MapPoint::MapPoint(const double invDepth, cv::Point2f uv_init, KeyFrame* pRefKF, KeyFrame* pHostKF, Map* pMap): mnFirstKFid(pRefKF->mnId), mnFirstFrame(pRefKF->mnFrameId), nObs(0), mnTrackReferenceForFrame(0), mnLastFrameSeen(0), mnBALocalForKF(0), mnFuseCandidateForKF(0), mnLoopPointForKF(0), mnCorrectedByKF(0), mnCorrectedReference(0), mnBAGlobalForKF(0), mpRefKF(pRefKF), mnVisible(1), mnFound(1), mbBad(false), mpReplaced(static_cast(NULL)), mfMinDistance(0), mfMaxDistance(0), mpMap(pMap), mnOriginMapId(pMap->GetId()) { mInvDepth=invDepth; mInitU=(double)uv_init.x; mInitV=(double)uv_init.y; mpHostKF = pHostKF; mNormalVector = cv::Mat::zeros(3,1,CV_32F); mNormalVectorx = cv::Matx31f::zeros(); // Worldpos is not set // MapPoints can be created from Tracking and Local Mapping. This mutex avoid conflicts with id. unique_lock lock(mpMap->mMutexPointCreation); mnId=nNextId++; } MapPoint::MapPoint(const cv::Mat &Pos, Map* pMap, Frame* pFrame, const int &idxF): mnFirstKFid(-1), mnFirstFrame(pFrame->mnId), nObs(0), mnTrackReferenceForFrame(0), mnLastFrameSeen(0), mnBALocalForKF(0), mnFuseCandidateForKF(0),mnLoopPointForKF(0), mnCorrectedByKF(0), mnCorrectedReference(0), mnBAGlobalForKF(0), mpRefKF(static_cast(NULL)), mnVisible(1), mnFound(1), mbBad(false), mpReplaced(NULL), mpMap(pMap), mnOriginMapId(pMap->GetId()) { Pos.copyTo(mWorldPos); mWorldPosx = cv::Matx31f(Pos.at(0), Pos.at(1), Pos.at(2)); cv::Mat Ow; if(pFrame -> Nleft == -1 || idxF < pFrame -> Nleft){ Ow = pFrame->GetCameraCenter(); } else{ cv::Mat Rwl = pFrame -> mRwc; cv::Mat tlr = pFrame -> mTlr.col(3); cv::Mat twl = pFrame -> mOw; Ow = Rwl * tlr + twl; } mNormalVector = mWorldPos - Ow; mNormalVector = mNormalVector/cv::norm(mNormalVector); mNormalVectorx = cv::Matx31f(mNormalVector.at(0), mNormalVector.at(1), mNormalVector.at(2)); cv::Mat PC = Pos - Ow; const float dist = cv::norm(PC); const int level = (pFrame -> Nleft == -1) ? pFrame->mvKeysUn[idxF].octave : (idxF < pFrame -> Nleft) ? pFrame->mvKeys[idxF].octave : pFrame -> mvKeysRight[idxF].octave; const float levelScaleFactor = pFrame->mvScaleFactors[level]; const int nLevels = pFrame->mnScaleLevels; mfMaxDistance = dist*levelScaleFactor; mfMinDistance = mfMaxDistance/pFrame->mvScaleFactors[nLevels-1]; pFrame->mDescriptors.row(idxF).copyTo(mDescriptor); // MapPoints can be created from Tracking and Local Mapping. This mutex avoid conflicts with id. unique_lock lock(mpMap->mMutexPointCreation); mnId=nNextId++; } void MapPoint::SetWorldPos(const cv::Mat &Pos) { unique_lock lock2(mGlobalMutex); unique_lock lock(mMutexPos); Pos.copyTo(mWorldPos); mWorldPosx = cv::Matx31f(Pos.at(0), Pos.at(1), Pos.at(2)); } cv::Mat MapPoint::GetWorldPos() { unique_lock lock(mMutexPos); return mWorldPos.clone(); } cv::Mat MapPoint::GetNormal() { unique_lock lock(mMutexPos); return mNormalVector.clone(); } cv::Matx31f MapPoint::GetWorldPos2() { unique_lock lock(mMutexPos); return mWorldPosx; } cv::Matx31f MapPoint::GetNormal2() { unique_lock lock(mMutexPos); return mNormalVectorx; } KeyFrame* MapPoint::GetReferenceKeyFrame() { unique_lock lock(mMutexFeatures); return mpRefKF; } /** * @brief 给地图点添加观测 * * 记录哪些 KeyFrame 的那个特征点能观测到该 地图点 * 并增加观测的相机数目nObs,单目+1,双目或者rgbd+2 * 这个函数是建立关键帧共视关系的核心函数,能共同观测到某些地图点的关键帧是共视关键帧 * @param pKF KeyFrame * @param idx MapPoint在KeyFrame中的索引 */ void MapPoint::AddObservation(KeyFrame* pKF, int idx) { unique_lock lock(mMutexFeatures); tuple indexes; if(mObservations.count(pKF)){ indexes = mObservations[pKF]; } else{ indexes = tuple(-1,-1); } if(pKF -> NLeft != -1 && idx >= pKF -> NLeft){ get<1>(indexes) = idx; } else{ get<0>(indexes) = idx; } // 如果没有添加过观测,记录下能观测到该MapPoint的KF和该MapPoint在KF中的索引 mObservations[pKF]=indexes; if(!pKF->mpCamera2 && pKF->mvuRight[idx]>=0) nObs+=2; // 双目或者rgbd else nObs++; // 单目 } // 删除某个关键帧对当前地图点的观测 void MapPoint::EraseObservation(KeyFrame* pKF) { bool bBad=false; { unique_lock lock(mMutexFeatures); // 查找这个要删除的观测,根据单目和双目类型的不同从其中删除当前地图点的被观测次数 if(mObservations.count(pKF)) { tuple indexes = mObservations[pKF]; int leftIndex = get<0>(indexes), rightIndex = get<1>(indexes); if(leftIndex != -1){ if(!pKF->mpCamera2 && pKF->mvuRight[leftIndex]>=0) nObs-=2; else nObs--; } if(rightIndex != -1){ nObs--; } mObservations.erase(pKF); // 如果该keyFrame是参考帧,该Frame被删除后重新指定RefFrame if(mpRefKF==pKF) mpRefKF=mObservations.begin()->first; // If only 2 observations or less, discard point // 当观测到该点的相机数目少于2时,丢弃该点 if(nObs<=2) bBad=true; } } if(bBad) // 告知可以观测到该MapPoint的Frame,该MapPoint已被删除 SetBadFlag(); } // 能够观测到当前地图点的所有关键帧及该地图点在KF中的索引 std::map> MapPoint::GetObservations() { unique_lock lock(mMutexFeatures); return mObservations; } // 被观测到的相机数目,单目+1,双目或RGB-D则+2 int MapPoint::Observations() { unique_lock lock(mMutexFeatures); return nObs; } /** * @brief 告知可以观测到该MapPoint的Frame,该MapPoint已被删除 * */ void MapPoint::SetBadFlag() { map> obs; { unique_lock lock1(mMutexFeatures); unique_lock lock2(mMutexPos); mbBad=true; // 把mObservations转存到obs,obs和mObservations里存的是指针,赋值过程为浅拷贝 obs = mObservations; // 把mObservations指向的内存释放,obs作为局部变量之后自动删除 mObservations.clear(); } for(map>::iterator mit=obs.begin(), mend=obs.end(); mit!=mend; mit++) { KeyFrame* pKF = mit->first; int leftIndex = get<0>(mit -> second), rightIndex = get<1>(mit -> second); if(leftIndex != -1){ pKF->EraseMapPointMatch(leftIndex); } if(rightIndex != -1){ pKF->EraseMapPointMatch(rightIndex); } } // 擦除该MapPoint申请的内存 mpMap->EraseMapPoint(this); } MapPoint* MapPoint::GetReplaced() { unique_lock lock1(mMutexFeatures); unique_lock lock2(mMutexPos); return mpReplaced; } /** * @brief 替换地图点,更新观测关系 * * @param[in] pMP 用该地图点来替换当前地图点 */ void MapPoint::Replace(MapPoint* pMP) { // 同一个地图点则跳过 if(pMP->mnId==this->mnId) return; //要替换当前地图点,有两个工作: // 1. 将当前地图点的观测数据等其他数据都"叠加"到新的地图点上 // 2. 将观测到当前地图点的关键帧的信息进行更新 // 清除当前地图点的信息,这一段和SetBadFlag函数相同 int nvisible, nfound; map> obs; { unique_lock lock1(mMutexFeatures); unique_lock lock2(mMutexPos); obs=mObservations; //清除当前地图点的原有观测 mObservations.clear(); //当前的地图点被删除了 mbBad=true; //暂存当前地图点的可视次数和被找到的次数 nvisible = mnVisible; nfound = mnFound; //指明当前地图点已经被指定的地图点替换了 mpReplaced = pMP; } // 所有能观测到原地图点的关键帧都要复制到替换的地图点上 for(map>::iterator mit=obs.begin(), mend=obs.end(); mit!=mend; mit++) { // Replace measurement in keyframe KeyFrame* pKF = mit->first; tuple indexes = mit -> second; int leftIndex = get<0>(indexes), rightIndex = get<1>(indexes); if(!pMP->IsInKeyFrame(pKF)) { if(leftIndex != -1){ pKF->ReplaceMapPointMatch(leftIndex, pMP); pMP->AddObservation(pKF,leftIndex); } if(rightIndex != -1){ pKF->ReplaceMapPointMatch(rightIndex, pMP); pMP->AddObservation(pKF,rightIndex); } } else { if(leftIndex != -1){ pKF->EraseMapPointMatch(leftIndex); } if(rightIndex != -1){ pKF->EraseMapPointMatch(rightIndex); } } } //- 将当前地图点的观测数据等其他数据都"叠加"到新的地图点上 pMP->IncreaseFound(nfound); pMP->IncreaseVisible(nvisible); //描述子更新 pMP->ComputeDistinctiveDescriptors(); //告知地图,删掉我 mpMap->EraseMapPoint(this); } bool MapPoint::isBad() { unique_lock lock1(mMutexFeatures,std::defer_lock); unique_lock lock2(mMutexPos,std::defer_lock); lock(lock1, lock2); return mbBad; } void MapPoint::IncreaseVisible(int n) { unique_lock lock(mMutexFeatures); mnVisible+=n; } void MapPoint::IncreaseFound(int n) { unique_lock lock(mMutexFeatures); mnFound+=n; } // 计算被找到的比例 float MapPoint::GetFoundRatio() { unique_lock lock(mMutexFeatures); return static_cast(mnFound)/mnVisible; } /** * @brief 计算地图点具有代表性的描述子 * * 由于一个MapPoint会被许多相机观测到,因此在插入关键帧后,需要判断是否更新当前点的最适合的描述子 * 先获得当前点的所有描述子,然后计算描述子之间的两两距离,最好的描述子与其他描述子应该具有最小的距离中值 */ void MapPoint::ComputeDistinctiveDescriptors() { // Retrieve all observed descriptors vector vDescriptors; map> observations; // Step 1 获取所有观测,跳过坏点 { unique_lock lock1(mMutexFeatures); if(mbBad) return; observations=mObservations; } if(observations.empty()) return; vDescriptors.reserve(observations.size()); // Step 2 遍历观测到3d点的所有关键帧,获得orb描述子,并插入到vDescriptors中 for(map>::iterator mit=observations.begin(), mend=observations.end(); mit!=mend; mit++) { // mit->first取观测到该地图点的关键帧 // mit->second取该地图点在关键帧中的索引 KeyFrame* pKF = mit->first; if(!pKF->isBad()){ tuple indexes = mit -> second; int leftIndex = get<0>(indexes), rightIndex = get<1>(indexes); if(leftIndex != -1){ vDescriptors.push_back(pKF->mDescriptors.row(leftIndex)); } if(rightIndex != -1){ vDescriptors.push_back(pKF->mDescriptors.row(rightIndex)); } } } if(vDescriptors.empty()) return; // Compute distances between them // Step 3 获得这些描述子两两之间的距离 // N表示为一共多少个描述子 const size_t N = vDescriptors.size(); float Distances[N][N]; for(size_t i=0;i vDists(Distances[i],Distances[i]+N); sort(vDists.begin(),vDists.end()); // 获得中值 int median = vDists[0.5*(N-1)]; // 寻找最小的中值 if(median lock(mMutexFeatures); mDescriptor = vDescriptors[BestIdx].clone(); } } //获取当前地图点的描述子 cv::Mat MapPoint::GetDescriptor() { unique_lock lock(mMutexFeatures); return mDescriptor.clone(); } //获取当前地图点在某个关键帧的观测中,对应的特征点的ID tuple MapPoint::GetIndexInKeyFrame(KeyFrame *pKF) { unique_lock lock(mMutexFeatures); if(mObservations.count(pKF)) return mObservations[pKF]; else return tuple(-1,-1); } bool MapPoint::IsInKeyFrame(KeyFrame *pKF) { unique_lock lock(mMutexFeatures); // 存在返回true,不存在返回false // std::map.count 用法见:http://www.cplusplus.com/reference/map/map/count/ return (mObservations.count(pKF)); } /** * @brief 更新平均观测方向以及观测距离范围 * * 由于一个MapPoint会被许多相机观测到,因此在插入关键帧后,需要更新相应变量 * 创建新的关键帧的时候会调用 */ void MapPoint::UpdateNormalAndDepth() { // Step 1 获得该地图点的相关信息 map> observations; KeyFrame* pRefKF; cv::Mat Pos; { unique_lock lock1(mMutexFeatures); unique_lock lock2(mMutexPos); if(mbBad) return; observations=mObservations; // 获得观测到该地图点的所有关键帧 pRefKF=mpRefKF; // 观测到该点的参考关键帧(第一次创建时的关键帧) Pos = mWorldPos.clone(); // 地图点在世界坐标系中的位置 } if(observations.empty()) return; // Step 2 计算该地图点的法线方向,也就是朝向等信息。 // 能观测到该地图点的所有关键帧,对该点的观测方向归一化为单位向量,然后进行求和得到该地图点的朝向 // 初始值为0向量,累加为归一化向量,最后除以总数n cv::Mat normal = cv::Mat::zeros(3,1,CV_32F); int n=0; for(map>::iterator mit=observations.begin(), mend=observations.end(); mit!=mend; mit++) { KeyFrame* pKF = mit->first; tuple indexes = mit -> second; int leftIndex = get<0>(indexes), rightIndex = get<1>(indexes); if(leftIndex != -1){ cv::Mat Owi = pKF->GetCameraCenter(); cv::Mat normali = mWorldPos - Owi; normal = normal + normali/cv::norm(normali); n++; } if(rightIndex != -1){ cv::Mat Owi = pKF->GetRightCameraCenter(); cv::Mat normali = mWorldPos - Owi; normal = normal + normali/cv::norm(normali); n++; } } cv::Mat PC = Pos - pRefKF->GetCameraCenter(); // 参考关键帧相机指向地图点的向量(在世界坐标系下的表示) const float dist = cv::norm(PC); // 该点到参考关键帧相机的距离 tuple indexes = observations[pRefKF]; int leftIndex = get<0>(indexes), rightIndex = get<1>(indexes); int level; if(pRefKF -> NLeft == -1){ level = pRefKF->mvKeysUn[leftIndex].octave; } else if(leftIndex != -1){ level = pRefKF -> mvKeys[leftIndex].octave; } else{ level = pRefKF -> mvKeysRight[rightIndex - pRefKF -> NLeft].octave; } //const int level = pRefKF->mvKeysUn[observations[pRefKF]].octave; const float levelScaleFactor = pRefKF->mvScaleFactors[level]; // 当前金字塔层对应的缩放倍数 const int nLevels = pRefKF->mnScaleLevels; // 金字塔层数 { unique_lock lock3(mMutexPos); // 使用方法见PredictScale函数前的注释 mfMaxDistance = dist*levelScaleFactor; // 观测到该点的距离上限 mfMinDistance = mfMaxDistance/pRefKF->mvScaleFactors[nLevels-1]; // 观测到该点的距离下限 mNormalVector = normal/n; // 获得地图点平均的观测方向 mNormalVectorx = cv::Matx31f(mNormalVector.at(0), mNormalVector.at(1), mNormalVector.at(2)); } } void MapPoint::SetNormalVector(cv::Mat& normal) { unique_lock lock3(mMutexPos); mNormalVector = normal; mNormalVectorx = cv::Matx31f(mNormalVector.at(0), mNormalVector.at(1), mNormalVector.at(2)); } float MapPoint::GetMinDistanceInvariance() { unique_lock lock(mMutexPos); return 0.8f*mfMinDistance; } float MapPoint::GetMaxDistanceInvariance() { unique_lock lock(mMutexPos); return 1.2f*mfMaxDistance; } // 下图中横线的大小表示不同图层图像上的一个像素表示的真实物理空间中的大小 // ____ // Nearer /____\ level:n-1 --> dmin // /______\ d/dmin = 1.2^(n-1-m) // /________\ level:m --> d // /__________\ dmax/d = 1.2^m // Farther /____________\ level:0 --> dmax // // log(dmax/d) // m = ceil(------------) // log(1.2) // 这个函数的作用: // 在进行投影匹配的时候会给定特征点的搜索范围,考虑到处于不同尺度(也就是距离相机远近,位于图像金字塔中不同图层)的特征点受到相机旋转的影响不同, // 因此会希望距离相机近的点的搜索范围更大一点,距离相机更远的点的搜索范围更小一点,所以要在这里,根据点到关键帧/帧的距离来估计它在当前的关键帧/帧中, // 会大概处于哪个尺度 int MapPoint::PredictScale(const float ¤tDist, KeyFrame* pKF) { float ratio; { unique_lock lock(mMutexPos); // mfMaxDistance = ref_dist*levelScaleFactor 为参考帧考虑上尺度后的距离 // ratio = mfMaxDistance/currentDist = ref_dist/cur_dist ratio = mfMaxDistance/currentDist; } // 同时取log线性化 int nScale = ceil(log(ratio)/pKF->mfLogScaleFactor); if(nScale<0) nScale = 0; else if(nScale>=pKF->mnScaleLevels) nScale = pKF->mnScaleLevels-1; return nScale; } /** * @brief 根据地图点到光心的距离来预测一个类似特征金字塔的尺度 * * @param[in] currentDist 地图点到光心的距离 * @param[in] pF 当前帧 * @return int 尺度 */ int MapPoint::PredictScale(const float ¤tDist, Frame* pF) { float ratio; { unique_lock lock(mMutexPos); ratio = mfMaxDistance/currentDist; } int nScale = ceil(log(ratio)/pF->mfLogScaleFactor); if(nScale<0) nScale = 0; else if(nScale>=pF->mnScaleLevels) nScale = pF->mnScaleLevels-1; return nScale; } Map* MapPoint::GetMap() { unique_lock lock(mMutexMap); return mpMap; } void MapPoint::UpdateMap(Map* pMap) { unique_lock lock(mMutexMap); mpMap = pMap; } } //namespace ORB_SLAM