From 3b6661e6100768d00d9f1faf60cfaf151071b50a Mon Sep 17 00:00:00 2001 From: kalwalt Date: Wed, 23 Sep 2026 10:35:33 +0200 Subject: [PATCH 1/2] feat(kpm): report every matched page, not only the best one VisualDatabase::query() now keeps every reference image that passes the inlier tests, exposed as matches(); matchedId()/inliers() still give the best. kpmMatching writes one pose per page, keeping the best-supported image of each page, and the dead per-page loop is removed. Refs webarkit/jsartoolkitNFT#635 Co-Authored-By: Claude Sonnet 5 --- .../facade/visual_database_facade.cpp | 6 +- .../facade/visual_database_facade.h | 4 +- .../KPM/FreakMatcher/matchers/matcher_types.h | 13 +++- .../matchers/visual_database-inline.h | 21 ++++-- .../FreakMatcher/matchers/visual_database.h | 12 ++- lib/SRC/KPM/kpmMatching.cpp | 74 ++++++++----------- 6 files changed, 78 insertions(+), 52 deletions(-) diff --git a/lib/SRC/KPM/FreakMatcher/facade/visual_database_facade.cpp b/lib/SRC/KPM/FreakMatcher/facade/visual_database_facade.cpp index 2b4ba88..c943083 100644 --- a/lib/SRC/KPM/FreakMatcher/facade/visual_database_facade.cpp +++ b/lib/SRC/KPM/FreakMatcher/facade/visual_database_facade.cpp @@ -150,7 +150,11 @@ namespace vision { const matches_t& VisualDatabaseFacade::inliers() const{ return mVisualDbImpl->mVdb->inliers(); } - + + const image_matches_t& VisualDatabaseFacade::matches() const{ + return mVisualDbImpl->mVdb->matches(); + } + int VisualDatabaseFacade::getWidth(int image_id) const{ return mVisualDbImpl->mVdb->keyframe(image_id)->width(); } diff --git a/lib/SRC/KPM/FreakMatcher/facade/visual_database_facade.h b/lib/SRC/KPM/FreakMatcher/facade/visual_database_facade.h index a60a1c6..8d66c32 100644 --- a/lib/SRC/KPM/FreakMatcher/facade/visual_database_facade.h +++ b/lib/SRC/KPM/FreakMatcher/facade/visual_database_facade.h @@ -91,7 +91,9 @@ namespace vision { const std::vector& getQueryDescriptors() const; const matches_t& inliers() const; - + + const image_matches_t& matches() const; + private: std::unique_ptr mVisualDbImpl; }; // VisualDatabaseFacade diff --git a/lib/SRC/KPM/FreakMatcher/matchers/matcher_types.h b/lib/SRC/KPM/FreakMatcher/matchers/matcher_types.h index 99f1b98..f37418b 100644 --- a/lib/SRC/KPM/FreakMatcher/matchers/matcher_types.h +++ b/lib/SRC/KPM/FreakMatcher/matchers/matcher_types.h @@ -47,5 +47,16 @@ namespace vision { }; // match_t typedef std::vector matches_t; - + + /** + * One reference image that survived geometric verification against the query. + */ + struct image_match_t { + int id; // reference image id: the db_id passed to addImage() + matches_t inliers; // correspondences consistent with `geometry` + float geometry[9]; // homography, row-major + }; // image_match_t + + typedef std::vector image_matches_t; + } // vision \ No newline at end of file diff --git a/lib/SRC/KPM/FreakMatcher/matchers/visual_database-inline.h b/lib/SRC/KPM/FreakMatcher/matchers/visual_database-inline.h index df05511..85c81c8 100644 --- a/lib/SRC/KPM/FreakMatcher/matchers/visual_database-inline.h +++ b/lib/SRC/KPM/FreakMatcher/matchers/visual_database-inline.h @@ -193,7 +193,8 @@ namespace vision { bool VisualDatabase::query(const keyframe_t* query_keyframe) { mMatchedInliers.clear(); mMatchedId = -1; - + mMatches.clear(); + const std::vector& query_points = query_keyframe->store().points(); // Loop over all the images in the database @@ -337,10 +338,20 @@ namespace vision { } //std::cout<<"inliers-"<= mMinNumInliers && inliers.size() > mMatchedInliers.size()) { - CopyVector9(mMatchedGeometry, H); - mMatchedInliers.swap(inliers); - mMatchedId = it->first; + if(inliers.size() >= mMinNumInliers) { + // Keep every image that passes, not only the best (#635): with + // several markers in view, each one has its own match. + image_match_t match; + match.id = it->first; + match.inliers = inliers; + CopyVector9(match.geometry, H); + mMatches.push_back(match); + + if(inliers.size() > mMatchedInliers.size()) { + CopyVector9(mMatchedGeometry, H); + mMatchedInliers.swap(inliers); + mMatchedId = it->first; + } } } diff --git a/lib/SRC/KPM/FreakMatcher/matchers/visual_database.h b/lib/SRC/KPM/FreakMatcher/matchers/visual_database.h index 5d41ecc..db20996 100644 --- a/lib/SRC/KPM/FreakMatcher/matchers/visual_database.h +++ b/lib/SRC/KPM/FreakMatcher/matchers/visual_database.h @@ -159,7 +159,14 @@ namespace vision { * @return Matched geometry matrix */ const float* matchedGeometry() const { return mMatchedGeometry; } - + + /** + * @return Every reference image that passed the inlier tests in the last + * query(), in database order. matchedId()/inliers()/matchedGeometry() + * still describe the single best of these. + */ + const image_matches_t& matches() const { return mMatches; } + /** * Get the detector. */ @@ -183,7 +190,8 @@ namespace vision { matches_t mMatchedInliers; id_t mMatchedId; float mMatchedGeometry[9]; - + image_matches_t mMatches; + keyframe_ptr_t mQueryKeyframe; // Map of keyframe diff --git a/lib/SRC/KPM/kpmMatching.cpp b/lib/SRC/KPM/kpmMatching.cpp index 9c50e07..8d5dd19 100644 --- a/lib/SRC/KPM/kpmMatching.cpp +++ b/lib/SRC/KPM/kpmMatching.cpp @@ -39,6 +39,7 @@ #include #include #include +#include #include #include "kpmPrivate.h" @@ -638,50 +639,39 @@ for (int pageLoop = 0; pageLoop < kpmHandle->resultNum; pageLoop++) { kpmHandle->result[pageLoop].camPoseF = -1; } -const vision::matches_t& matches = kpmHandle->freakMatcher->inliers(); -int matched_image_id = kpmHandle->freakMatcher->matchedId(); -ARLOGd("kpmMatching: matchedId=%d (resultNum=%d, inliers=%d)\n", matched_image_id, kpmHandle->resultNum, (int)matches.size()); -if (matched_image_id >= 0) { - int matchedPageNo = kpmHandle->pageIDs[matched_image_id]; - ARLOGd("kpmMatching: matchedId=%d maps to pageNo=%d\n", matched_image_id, matchedPageNo); - - if( !kpmHandle->result[matchedPageNo].skipF ) { - ret = kpmUtilGetPose_binary(kpmHandle->cparamLT, - matches , - kpmHandle->freakMatcher->get3DFeaturePoints(matched_image_id), - kpmHandle->freakMatcher->getQueryFeaturePoints(), - kpmHandle->result[matchedPageNo].camPose, - &(kpmHandle->result[matchedPageNo].error) ); - - if (ret == 0) { - kpmHandle->result[matchedPageNo].camPoseF = 0; - kpmHandle->result[matchedPageNo].inlierNum = (int)matches.size(); - kpmHandle->result[matchedPageNo].pageNo = matchedPageNo; - ARLOGi("Page[%d] pre:%3d, aft:%3d, error = %f\n", matchedPageNo, (int)matches.size(), (int)matches.size(), kpmHandle->result[matchedPageNo].error); - } +// Every reference image that passed the matcher's tests (#635). Each page was +// registered as several images — one per scale, see kpmSetRefDataSet — so +// several matches can belong to one page. Keep the best-supported per page. +const vision::image_matches_t& imageMatches = kpmHandle->freakMatcher->matches(); +std::map bestPerPage; +for (const vision::image_match_t& imageMatch : imageMatches) { + const int pageNo = kpmHandle->pageIDs[imageMatch.id]; + auto best = bestPerPage.find(pageNo); + if (best == bestPerPage.end() || imageMatch.inliers.size() > best->second->inliers.size()) { + bestPerPage[pageNo] = &imageMatch; + } +} +ARLOGd("kpmMatching: %d image match(es) across %d page(s)\n", (int)imageMatches.size(), (int)bestPerPage.size()); + +for (const auto& entry : bestPerPage) { + const int pageNo = entry.first; + const vision::image_match_t& imageMatch = *entry.second; + // result[] is indexed by page number, as it was before this change. + if (kpmHandle->result[pageNo].skipF) continue; + + ret = kpmUtilGetPose_binary(kpmHandle->cparamLT, + imageMatch.inliers, + kpmHandle->freakMatcher->get3DFeaturePoints(imageMatch.id), + kpmHandle->freakMatcher->getQueryFeaturePoints(), + kpmHandle->result[pageNo].camPose, + &(kpmHandle->result[pageNo].error)); + if (ret == 0) { + kpmHandle->result[pageNo].camPoseF = 0; + kpmHandle->result[pageNo].inlierNum = (int)imageMatch.inliers.size(); + kpmHandle->result[pageNo].pageNo = pageNo; + ARLOGi("Page[%d] pre:%3d, aft:%3d, error = %f\n", pageNo, (int)imageMatch.inliers.size(), (int)imageMatch.inliers.size(), kpmHandle->result[pageNo].error); } } -/* - for (int pageLoop = 0; pageLoop < kpmHandle->resultNum; pageLoop++) { - - kpmHandle->result[pageLoop].pageNo = kpmHandle->refDataSet.pageInfo[pageLoop].pageNo; - kpmHandle->result[pageLoop].camPoseF = -1; - if( kpmHandle->result[pageLoop].skipF ) continue; - - - const vision::matches_t& matches = kpmHandle->freakMatcher->inliers(); - int matched_image_id = kpmHandle->freakMatcher->matchedId(); - if (matched_image_id < 0) continue; - - //ARLOGi("Pose (freak) - %s",arrayToString2(kpmHandle->result[pageLoop].camPose).c_str()); - if( ret == 0 ) { - kpmHandle->result[pageLoop].camPoseF = 0; - kpmHandle->result[pageLoop].inlierNum = (int)matches.size(); - kpmHandle->result[pageLoop].pageNo = kpmHandle->pageIDs[matched_image_id]; - ARLOGi("Page[%d] pre:%3d, aft:%3d, error = %f\n", pageLoop, (int)matches.size(), (int)matches.size(), kpmHandle->result[pageLoop].error); - } - } - */ #endif #if !BINARY_FEATURE free(featureVector.sf); From f3d4959927cd87ea4e3a5b4a50b2387cefd80d27 Mon Sep 17 00:00:00 2001 From: kalwalt Date: Wed, 23 Sep 2026 17:04:26 +0200 Subject: [PATCH 2/2] fix(kpm): index results by page position and fall back across a page's images Two review findings on #71: - result[] is allocated densely, one slot per position in refDataSet.pageInfo, and kpmSetMatchingSkipPage() indexes it that way, but the per-page loop indexed it by the page's external number. Those only coincide when pages are numbered 0..n-1 in load order; kpmChangePageNoOfRefDataSet() can set any value. Record each db_id's page position (pageIndices) and index result[] by it, reporting the external pageNo in KpmResult::pageNo as before. - If the pose of a page's best-supported image could not be fitted, the page was dropped even when another verified image of the same page would give a pose. Try the page's images in descending inlier order and keep the first that succeeds. No extra cost when the first succeeds. Refs webarkit/jsartoolkitNFT#635 Co-Authored-By: Claude Opus 5.5 --- lib/SRC/KPM/kpmMatching.cpp | 62 +++++++++++++++++++++---------------- lib/SRC/KPM/kpmPrivate.h | 1 + 2 files changed, 37 insertions(+), 26 deletions(-) diff --git a/lib/SRC/KPM/kpmMatching.cpp b/lib/SRC/KPM/kpmMatching.cpp index 8d5dd19..4246ace 100644 --- a/lib/SRC/KPM/kpmMatching.cpp +++ b/lib/SRC/KPM/kpmMatching.cpp @@ -40,6 +40,7 @@ #include #include #include +#include #include #include "kpmPrivate.h" @@ -294,6 +295,7 @@ int kpmSetRefDataSet( KpmHandle *kpmHandle, KpmRefDataSet *refDataSet ) } ARLOGi("page %d, image num %d, points - %d\n", k, m, points.size()); kpmHandle->pageIDs[db_id] = kpmHandle->refDataSet.pageInfo[k].pageNo; + kpmHandle->pageIndices[db_id] = k; ARLOGd("kpmSetRefDataSet: db_id=%d -> pageNo=%d (page index %d, image %d, %d points)\n", db_id, kpmHandle->refDataSet.pageInfo[k].pageNo, k, m, (int)points.size()); kpmHandle->freakMatcher->addFreakFeaturesAndDescriptors(points,descriptors,points_3d,kpmHandle->refDataSet.pageInfo[k].imageInfo[m].width,kpmHandle->refDataSet.pageInfo[k].imageInfo[m].height,db_id++); } @@ -641,35 +643,43 @@ for (int pageLoop = 0; pageLoop < kpmHandle->resultNum; pageLoop++) { // Every reference image that passed the matcher's tests (#635). Each page was // registered as several images — one per scale, see kpmSetRefDataSet — so -// several matches can belong to one page. Keep the best-supported per page. +// several matches can belong to one page. +// +// result[] is indexed by the page's position in refDataSet.pageInfo, as +// kpmSetMatchingSkipPage() indexes it, NOT by the page's external number: +// page numbers are labels (kpmChangePageNoOfRefDataSet can set any value). const vision::image_matches_t& imageMatches = kpmHandle->freakMatcher->matches(); -std::map bestPerPage; +std::map > candidatesPerPage; for (const vision::image_match_t& imageMatch : imageMatches) { - const int pageNo = kpmHandle->pageIDs[imageMatch.id]; - auto best = bestPerPage.find(pageNo); - if (best == bestPerPage.end() || imageMatch.inliers.size() > best->second->inliers.size()) { - bestPerPage[pageNo] = &imageMatch; - } + candidatesPerPage[kpmHandle->pageIndices[imageMatch.id]].push_back(&imageMatch); } -ARLOGd("kpmMatching: %d image match(es) across %d page(s)\n", (int)imageMatches.size(), (int)bestPerPage.size()); - -for (const auto& entry : bestPerPage) { - const int pageNo = entry.first; - const vision::image_match_t& imageMatch = *entry.second; - // result[] is indexed by page number, as it was before this change. - if (kpmHandle->result[pageNo].skipF) continue; - - ret = kpmUtilGetPose_binary(kpmHandle->cparamLT, - imageMatch.inliers, - kpmHandle->freakMatcher->get3DFeaturePoints(imageMatch.id), - kpmHandle->freakMatcher->getQueryFeaturePoints(), - kpmHandle->result[pageNo].camPose, - &(kpmHandle->result[pageNo].error)); - if (ret == 0) { - kpmHandle->result[pageNo].camPoseF = 0; - kpmHandle->result[pageNo].inlierNum = (int)imageMatch.inliers.size(); - kpmHandle->result[pageNo].pageNo = pageNo; - ARLOGi("Page[%d] pre:%3d, aft:%3d, error = %f\n", pageNo, (int)imageMatch.inliers.size(), (int)imageMatch.inliers.size(), kpmHandle->result[pageNo].error); +ARLOGd("kpmMatching: %d image match(es) across %d page(s)\n", (int)imageMatches.size(), (int)candidatesPerPage.size()); + +for (auto& entry : candidatesPerPage) { + const int pageIndex = entry.first; + KpmResult& result = kpmHandle->result[pageIndex]; + if (result.skipF) continue; + + // Try the page's best-supported image first. If its pose cannot be fitted, + // fall back to the page's other verified images rather than dropping the page. + std::vector& candidates = entry.second; + std::stable_sort(candidates.begin(), candidates.end(), + [](const vision::image_match_t* a, const vision::image_match_t* b) { + return a->inliers.size() > b->inliers.size(); + }); + for (const vision::image_match_t* imageMatch : candidates) { + ret = kpmUtilGetPose_binary(kpmHandle->cparamLT, + imageMatch->inliers, + kpmHandle->freakMatcher->get3DFeaturePoints(imageMatch->id), + kpmHandle->freakMatcher->getQueryFeaturePoints(), + result.camPose, + &(result.error)); + if (ret != 0) continue; + result.camPoseF = 0; + result.inlierNum = (int)imageMatch->inliers.size(); + result.pageNo = kpmHandle->refDataSet.pageInfo[pageIndex].pageNo; + ARLOGi("Page[%d] pre:%3d, aft:%3d, error = %f\n", result.pageNo, (int)imageMatch->inliers.size(), (int)imageMatch->inliers.size(), result.error); + break; } } #endif diff --git a/lib/SRC/KPM/kpmPrivate.h b/lib/SRC/KPM/kpmPrivate.h index 8440149..bc87681 100644 --- a/lib/SRC/KPM/kpmPrivate.h +++ b/lib/SRC/KPM/kpmPrivate.h @@ -88,6 +88,7 @@ struct _KpmHandle { KpmResult *result; int resultNum; int pageIDs[DB_IMAGE_MAX]; + int pageIndices[DB_IMAGE_MAX]; // position in refDataSet.pageInfo (and result[]) of each db_id's page }; #endif // !__kpmPrivate_h__