diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4c0dbb40..ac03f7de 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,11 +5,11 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Setup Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@v6 with: - node-version: '20.x' + node-version: '22.x' - name: Update Ubuntu and install libjpeg-dev run: | sudo apt-get update && sudo apt install libjpeg-dev @@ -20,7 +20,7 @@ jobs: run: | cd .. ls - docker run -dit --name emscripten-webarkit-testing -v $(pwd):/src emscripten/emsdk:3.1.38 bash + docker run -dit --name emscripten-webarkit-testing -v $(pwd):/src emscripten/emsdk:3.1.69 bash docker exec emscripten-webarkit-testing emcmake cmake -B WebARKitLib/WebARKit/build -S WebARKitLib/WebARKit -DEMSCRIPTEN_COMP=1 .. docker exec emscripten-webarkit-testing emmake make -C WebARKitLib/WebARKit/build \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2f7d9cdc..001968a9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,10 +6,27 @@ superproject as a submodule. ## Workflow -- **Branch from `dev`** and open your PR **against `dev`** (not `main`). +- **Branch from `dev`** and open your PR **against `dev`** (not `master`). - Reference the related issue in the PR description. - **Sign your commits** (`git commit -S …`). +### Merging a branch-sync PR + +Ordinary PRs can be squashed. A PR that **syncs one long-lived branch into another** +— typically `master` back into `dev` after a release — must be merged with +**"Create a merge commit"**, never squash or rebase. + +Squashing collapses the incoming commits into a single new commit, and rebasing +replays them under new SHAs. Either way `dev` ends up *containing* the code without +git recognising the two branches as related, so the next sync offers the same commits +again and usually conflicts. + +Only a merge commit makes the incoming commits genuine ancestors, which is what makes +the following sync a no-op. + +This is not hypothetical: `dev` drifted 14 commits behind `master` and had to be +repaired in #68, and consumers could not build against `dev` in the meantime. + ## Commit messages — Conventional Commits All commit messages **must** follow diff --git a/WebARKit/CMakeLists.txt b/WebARKit/CMakeLists.txt index 724e5cdd..1e6ea948 100644 --- a/WebARKit/CMakeLists.txt +++ b/WebARKit/CMakeLists.txt @@ -9,21 +9,33 @@ if(VERSION GREATER 3.24) endif() include(FetchContent) +include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/OpenCVEm.cmake) + +option(WEBARKIT_SIMD "Use the SIMD-enabled emscripten OpenCV build" OFF) if(${EMSCRIPTEN_COMP} EQUAL 1) - message("Fetching opencv for emscripten compilation from webarkit/opencv-em ...") - FetchContent_Declare( - build_opencv - URL https://github.com/webarkit/opencv-em/releases/download/0.1.6/opencv-js-4.10.0-emcc-3.1.38.zip - ) + if(WEBARKIT_SIMD) + set(OPENCV_FETCH_DESC "SIMD emscripten") + set(OPENCV_FETCH_URL ${OPENCV_EMSCRIPTEN_SIMD_URL}) + set(OPENCV_FETCH_HASH ${OPENCV_EMSCRIPTEN_SIMD_HASH}) + else() + set(OPENCV_FETCH_DESC "emscripten") + set(OPENCV_FETCH_URL ${OPENCV_EMSCRIPTEN_URL}) + set(OPENCV_FETCH_HASH ${OPENCV_EMSCRIPTEN_HASH}) + endif() else() - message("Fetching opencv from webarkit/opencv-em ...") - FetchContent_Declare( - build_opencv - URL https://github.com/webarkit/opencv-em/releases/download/0.1.6/opencv-4.10.0.zip - ) + set(OPENCV_FETCH_DESC "native") + set(OPENCV_FETCH_URL ${OPENCV_NATIVE_URL}) + set(OPENCV_FETCH_HASH ${OPENCV_NATIVE_HASH}) endif() +message("Fetching ${OPENCV_FETCH_DESC} opencv from webarkit/opencv-em ${OPENCV_EM_RELEASE} ...") +FetchContent_Declare( + build_opencv + URL ${OPENCV_FETCH_URL} + URL_HASH ${OPENCV_FETCH_HASH} +) + FetchContent_MakeAvailable(build_opencv) get_filename_component(PARENT_DIR ./ ABSOLUTE) diff --git a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/WebARKitConfig.cpp b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/WebARKitConfig.cpp index 1cf6e1e7..ec348da6 100644 --- a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/WebARKitConfig.cpp +++ b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/WebARKitConfig.cpp @@ -21,7 +21,7 @@ extern const cv::Size blurSize(3, 3); extern const double ransac_thresh = 2.5f; extern cv::RNG rng( 0xFFFFFFFF ); extern const double m_pi = 3.14159265358979323846; -extern const std::string WEBARKIT_HEADER_VERSION_STRING = "0.8.0"; +extern const std::string WEBARKIT_HEADER_VERSION_STRING = "0.9.0"; /*@ The MAJOR version number defines non-backwards compatible changes in the ARToolKit API. Range: [0-99]. @@ -33,7 +33,7 @@ extern const int WEBARKIT_HEADER_VERSION_MAJOR = 0; API, or (occsasionally) other significant backwards-compatible changes in runtime functionality. Range: [0-99]. */ -extern const int WEBARKIT_HEADER_VERSION_MINOR = 8; +extern const int WEBARKIT_HEADER_VERSION_MINOR = 9; /*@ The TINY version number defines bug-fixes to existing diff --git a/cmake/OpenCVEm.cmake b/cmake/OpenCVEm.cmake new file mode 100644 index 00000000..b3a1d402 --- /dev/null +++ b/cmake/OpenCVEm.cmake @@ -0,0 +1,13 @@ +# Shared webarkit/opencv-em release coordinates. +# Bump OPENCV_EM_RELEASE (and the three hashes below) to upgrade opencv-em. +set(OPENCV_EM_RELEASE "0.2.0") +set(OPENCV_EM_BASE_URL "https://github.com/webarkit/opencv-em/releases/download/${OPENCV_EM_RELEASE}") + +set(OPENCV_NATIVE_URL "${OPENCV_EM_BASE_URL}/opencv-4.12.0.zip") +set(OPENCV_NATIVE_HASH "SHA256=eb68b3c6cac2781f6bbbe747d9ac8f27c5d716471da82d6c4fd79f26a18263b4") + +set(OPENCV_EMSCRIPTEN_URL "${OPENCV_EM_BASE_URL}/opencv-js-4.12.0-emcc-3.1.69.zip") +set(OPENCV_EMSCRIPTEN_HASH "SHA256=3a9509615bed922b058e3201007c8b9b29c1e5aa3dd0750676b7d847738ce2c7") + +set(OPENCV_EMSCRIPTEN_SIMD_URL "${OPENCV_EM_BASE_URL}/opencv-js-4.12.0-emcc-3.1.69-simd.zip") +set(OPENCV_EMSCRIPTEN_SIMD_HASH "SHA256=3600fd9d0422cb1fc19306bb1f876a578dff970f207947b346180993dfa27026") diff --git a/lib/SRC/KPM/FreakMatcher/facade/visual_database_facade.cpp b/lib/SRC/KPM/FreakMatcher/facade/visual_database_facade.cpp index 2b4ba886..c9430830 100644 --- a/lib/SRC/KPM/FreakMatcher/facade/visual_database_facade.cpp +++ b/lib/SRC/KPM/FreakMatcher/facade/visual_database_facade.cpp @@ -150,7 +150,11 @@ namespace vision { const matches_t& VisualDatabaseFacade::inliers() const{ return mVisualDbImpl->mVdb->inliers(); } - + + const image_matches_t& VisualDatabaseFacade::matches() const{ + return mVisualDbImpl->mVdb->matches(); + } + int VisualDatabaseFacade::getWidth(int image_id) const{ return mVisualDbImpl->mVdb->keyframe(image_id)->width(); } diff --git a/lib/SRC/KPM/FreakMatcher/facade/visual_database_facade.h b/lib/SRC/KPM/FreakMatcher/facade/visual_database_facade.h index a60a1c6c..8d66c323 100644 --- a/lib/SRC/KPM/FreakMatcher/facade/visual_database_facade.h +++ b/lib/SRC/KPM/FreakMatcher/facade/visual_database_facade.h @@ -91,7 +91,9 @@ namespace vision { const std::vector& getQueryDescriptors() const; const matches_t& inliers() const; - + + const image_matches_t& matches() const; + private: std::unique_ptr mVisualDbImpl; }; // VisualDatabaseFacade diff --git a/lib/SRC/KPM/FreakMatcher/matchers/matcher_types.h b/lib/SRC/KPM/FreakMatcher/matchers/matcher_types.h index 99f1b98f..f37418b4 100644 --- a/lib/SRC/KPM/FreakMatcher/matchers/matcher_types.h +++ b/lib/SRC/KPM/FreakMatcher/matchers/matcher_types.h @@ -47,5 +47,16 @@ namespace vision { }; // match_t typedef std::vector matches_t; - + + /** + * One reference image that survived geometric verification against the query. + */ + struct image_match_t { + int id; // reference image id: the db_id passed to addImage() + matches_t inliers; // correspondences consistent with `geometry` + float geometry[9]; // homography, row-major + }; // image_match_t + + typedef std::vector image_matches_t; + } // vision \ No newline at end of file diff --git a/lib/SRC/KPM/FreakMatcher/matchers/visual_database-inline.h b/lib/SRC/KPM/FreakMatcher/matchers/visual_database-inline.h index df055114..85c81c87 100644 --- a/lib/SRC/KPM/FreakMatcher/matchers/visual_database-inline.h +++ b/lib/SRC/KPM/FreakMatcher/matchers/visual_database-inline.h @@ -193,7 +193,8 @@ namespace vision { bool VisualDatabase::query(const keyframe_t* query_keyframe) { mMatchedInliers.clear(); mMatchedId = -1; - + mMatches.clear(); + const std::vector& query_points = query_keyframe->store().points(); // Loop over all the images in the database @@ -337,10 +338,20 @@ namespace vision { } //std::cout<<"inliers-"<= mMinNumInliers && inliers.size() > mMatchedInliers.size()) { - CopyVector9(mMatchedGeometry, H); - mMatchedInliers.swap(inliers); - mMatchedId = it->first; + if(inliers.size() >= mMinNumInliers) { + // Keep every image that passes, not only the best (#635): with + // several markers in view, each one has its own match. + image_match_t match; + match.id = it->first; + match.inliers = inliers; + CopyVector9(match.geometry, H); + mMatches.push_back(match); + + if(inliers.size() > mMatchedInliers.size()) { + CopyVector9(mMatchedGeometry, H); + mMatchedInliers.swap(inliers); + mMatchedId = it->first; + } } } diff --git a/lib/SRC/KPM/FreakMatcher/matchers/visual_database.h b/lib/SRC/KPM/FreakMatcher/matchers/visual_database.h index 5d41ecc1..db209969 100644 --- a/lib/SRC/KPM/FreakMatcher/matchers/visual_database.h +++ b/lib/SRC/KPM/FreakMatcher/matchers/visual_database.h @@ -159,7 +159,14 @@ namespace vision { * @return Matched geometry matrix */ const float* matchedGeometry() const { return mMatchedGeometry; } - + + /** + * @return Every reference image that passed the inlier tests in the last + * query(), in database order. matchedId()/inliers()/matchedGeometry() + * still describe the single best of these. + */ + const image_matches_t& matches() const { return mMatches; } + /** * Get the detector. */ @@ -183,7 +190,8 @@ namespace vision { matches_t mMatchedInliers; id_t mMatchedId; float mMatchedGeometry[9]; - + image_matches_t mMatches; + keyframe_ptr_t mQueryKeyframe; // Map of keyframe diff --git a/lib/SRC/KPM/kpmMatching.cpp b/lib/SRC/KPM/kpmMatching.cpp index 51650253..28ec29bb 100644 --- a/lib/SRC/KPM/kpmMatching.cpp +++ b/lib/SRC/KPM/kpmMatching.cpp @@ -39,6 +39,8 @@ #include #include #include +#include +#include #include #include "kpmPrivate.h" @@ -187,6 +189,24 @@ int kpmSetRefDataSet( KpmHandle *kpmHandle, KpmRefDataSet *refDataSet ) ARLOGe("kpmSetRefDataSet(): refDataSet.\n"); return -1; } +#if BINARY_FEATURE + // pageIDs[] and pageIndices[] hold one entry per registered image (one per + // page per scale). Refuse a dataset that would not fit before changing any + // state, instead of writing past them. + { + // Check each count against the room left before adding it, so a corrupt + // or hostile dataset cannot overflow the running total past the check. + int imageTotal = 0; + for( i = 0; i < refDataSet->pageNum; i++ ) { + const int imageNum = refDataSet->pageInfo[i].imageNum; + if( imageNum < 0 || imageNum > DB_IMAGE_MAX - imageTotal ) { + ARLOGe("kpmSetRefDataSet(): page %d has %d reference images; the set exceeds DB_IMAGE_MAX (%d).\n", i, imageNum, DB_IMAGE_MAX); + return -1; + } + imageTotal += imageNum; + } + } +#endif // Copy the refPoints into the kpmHandle's dataset. if( kpmHandle->refDataSet.refPoint != NULL ) { @@ -267,6 +287,16 @@ int kpmSetRefDataSet( KpmHandle *kpmHandle, KpmRefDataSet *refDataSet ) free(featureVector.sf); } #else + // Register the new set in a fresh matcher. Reusing the old one kept every + // keyframe it already held: an id the new set registers again is refused + // as a duplicate (the old keyframe stays) while its 3D points are replaced, + // so pose estimation paired old matches with new, shorter point lists. The + // matcher carries no configuration beyond construction (kpmCreateHandle), + // so a new instance is equivalent to the original. + delete kpmHandle->freakMatcher; + kpmHandle->freakMatcher = new vision::VisualDatabaseFacade; + kpmHandle->dbImageNum = 0; + if (kpmHandle->refDataSet.num != 0) { featureVector.num = kpmHandle->refDataSet.num; @@ -293,9 +323,12 @@ int kpmSetRefDataSet( KpmHandle *kpmHandle, KpmRefDataSet *refDataSet ) } ARLOGi("page %d, image num %d, points - %d\n", k, m, points.size()); kpmHandle->pageIDs[db_id] = kpmHandle->refDataSet.pageInfo[k].pageNo; + kpmHandle->pageIndices[db_id] = k; + ARLOGd("kpmSetRefDataSet: db_id=%d -> pageNo=%d (page index %d, image %d, %d points)\n", db_id, kpmHandle->refDataSet.pageInfo[k].pageNo, k, m, (int)points.size()); kpmHandle->freakMatcher->addFreakFeaturesAndDescriptors(points,descriptors,points_3d,kpmHandle->refDataSet.pageInfo[k].imageInfo[m].width,kpmHandle->refDataSet.pageInfo[k].imageInfo[m].height,db_id++); } } + kpmHandle->dbImageNum = db_id; } #endif @@ -637,48 +670,59 @@ for (int pageLoop = 0; pageLoop < kpmHandle->resultNum; pageLoop++) { kpmHandle->result[pageLoop].camPoseF = -1; } -const vision::matches_t& matches = kpmHandle->freakMatcher->inliers(); -int matched_image_id = kpmHandle->freakMatcher->matchedId(); -if (matched_image_id != 0) { - int matchedPageNo = kpmHandle->pageIDs[matched_image_id]; - - if( !kpmHandle->result[matchedPageNo].skipF ) { +// Every reference image that passed the matcher's tests (#635). Each page was +// registered as several images — one per scale, see kpmSetRefDataSet — so +// several matches can belong to one page. +// +// result[] is indexed by the page's position in refDataSet.pageInfo, as +// kpmSetMatchingSkipPage() indexes it, NOT by the page's external number: +// page numbers are labels (kpmChangePageNoOfRefDataSet can set any value). +const vision::image_matches_t& imageMatches = kpmHandle->freakMatcher->matches(); +std::map > candidatesPerPage; +for (const vision::image_match_t& imageMatch : imageMatches) { + // Invariant checks. kpmSetRefDataSet() registers each set in a fresh + // matcher, so every id is below dbImageNum and maps into result[]; should + // that ever break, log it rather than index the mapping arrays out of bounds. + if (imageMatch.id < 0 || imageMatch.id >= kpmHandle->dbImageNum) { + ARLOGe("kpmMatching: matcher image %d is outside the current set (%d images).\n", imageMatch.id, kpmHandle->dbImageNum); + continue; + } + const int pageIndex = kpmHandle->pageIndices[imageMatch.id]; + if (pageIndex < 0 || pageIndex >= kpmHandle->resultNum) { + ARLOGe("kpmMatching: image %d maps to page position %d, outside 0..%d.\n", imageMatch.id, pageIndex, kpmHandle->resultNum - 1); + continue; + } + candidatesPerPage[pageIndex].push_back(&imageMatch); +} +ARLOGd("kpmMatching: %d image match(es) across %d page(s)\n", (int)imageMatches.size(), (int)candidatesPerPage.size()); + +for (auto& entry : candidatesPerPage) { + const int pageIndex = entry.first; + KpmResult& result = kpmHandle->result[pageIndex]; + if (result.skipF) continue; + + // Try the page's best-supported image first. If its pose cannot be fitted, + // fall back to the page's other verified images rather than dropping the page. + std::vector& candidates = entry.second; + std::stable_sort(candidates.begin(), candidates.end(), + [](const vision::image_match_t* a, const vision::image_match_t* b) { + return a->inliers.size() > b->inliers.size(); + }); + for (const vision::image_match_t* imageMatch : candidates) { ret = kpmUtilGetPose_binary(kpmHandle->cparamLT, - matches , - kpmHandle->freakMatcher->get3DFeaturePoints(matched_image_id), + imageMatch->inliers, + kpmHandle->freakMatcher->get3DFeaturePoints(imageMatch->id), kpmHandle->freakMatcher->getQueryFeaturePoints(), - kpmHandle->result[matchedPageNo].camPose, - &(kpmHandle->result[matchedPageNo].error) ); - - if (ret == 0) { - kpmHandle->result[matchedPageNo].camPoseF = 0; - kpmHandle->result[matchedPageNo].inlierNum = (int)matches.size(); - kpmHandle->result[matchedPageNo].pageNo = matchedPageNo; - ARLOGi("Page[%d] pre:%3d, aft:%3d, error = %f\n", matchedPageNo, (int)matches.size(), (int)matches.size(), kpmHandle->result[matchedPageNo].error); - } + result.camPose, + &(result.error)); + if (ret != 0) continue; + result.camPoseF = 0; + result.inlierNum = (int)imageMatch->inliers.size(); + result.pageNo = kpmHandle->refDataSet.pageInfo[pageIndex].pageNo; + ARLOGi("Page[%d] pre:%3d, aft:%3d, error = %f\n", result.pageNo, (int)imageMatch->inliers.size(), (int)imageMatch->inliers.size(), result.error); + break; } } -/* - for (int pageLoop = 0; pageLoop < kpmHandle->resultNum; pageLoop++) { - - kpmHandle->result[pageLoop].pageNo = kpmHandle->refDataSet.pageInfo[pageLoop].pageNo; - kpmHandle->result[pageLoop].camPoseF = -1; - if( kpmHandle->result[pageLoop].skipF ) continue; - - - const vision::matches_t& matches = kpmHandle->freakMatcher->inliers(); - int matched_image_id = kpmHandle->freakMatcher->matchedId(); - if (matched_image_id < 0) continue; - - //ARLOGi("Pose (freak) - %s",arrayToString2(kpmHandle->result[pageLoop].camPose).c_str()); - if( ret == 0 ) { - kpmHandle->result[pageLoop].camPoseF = 0; - kpmHandle->result[pageLoop].inlierNum = (int)matches.size(); - kpmHandle->result[pageLoop].pageNo = kpmHandle->pageIDs[matched_image_id]; - ARLOGi("Page[%d] pre:%3d, aft:%3d, error = %f\n", pageLoop, (int)matches.size(), (int)matches.size(), kpmHandle->result[pageLoop].error); - } - } - */ #endif #if !BINARY_FEATURE free(featureVector.sf); diff --git a/lib/SRC/KPM/kpmPrivate.h b/lib/SRC/KPM/kpmPrivate.h index 84401491..0b3a04a1 100644 --- a/lib/SRC/KPM/kpmPrivate.h +++ b/lib/SRC/KPM/kpmPrivate.h @@ -88,6 +88,8 @@ struct _KpmHandle { KpmResult *result; int resultNum; int pageIDs[DB_IMAGE_MAX]; + int pageIndices[DB_IMAGE_MAX]; // position in refDataSet.pageInfo (and result[]) of each db_id's page + int dbImageNum; // images registered by the last kpmSetRefDataSet(): ids 0..dbImageNum-1 are current }; #endif // !__kpmPrivate_h__ diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 503a7fb4..f6e4369f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -10,6 +10,7 @@ endif() # Fetch googletest v1.13.0 commit b796f7d44681514f58a683a3a71ff17c94edb0c1 include(FetchContent) +include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/OpenCVEm.cmake) FetchContent_Declare( googletest URL https://github.com/google/googletest/archive/b796f7d44681514f58a683a3a71ff17c94edb0c1.zip @@ -19,7 +20,8 @@ set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) FetchContent_Declare( build_opencv - URL https://github.com/webarkit/opencv-em/releases/download/0.1.6/opencv-4.10.0.zip + URL ${OPENCV_NATIVE_URL} + URL_HASH ${OPENCV_NATIVE_HASH} ) FetchContent_MakeAvailable(googletest build_opencv) diff --git a/tests/webarkit_test.cc b/tests/webarkit_test.cc index 7ac29641..a1553067 100644 --- a/tests/webarkit_test.cc +++ b/tests/webarkit_test.cc @@ -38,7 +38,7 @@ TEST(WebARKitConfigTest, TestConfigValues) { EXPECT_EQ(maxLevel, 3); EXPECT_EQ(featureDetectPyramidLevel, 1.05f); EXPECT_EQ(featureBorder, 8); - EXPECT_EQ(WEBARKIT_HEADER_VERSION_STRING, "0.8.0"); + EXPECT_EQ(WEBARKIT_HEADER_VERSION_STRING, "0.9.0"); } TEST(WebARKitConfigTest, TestWinSize) { @@ -136,7 +136,7 @@ TEST(WebARKitTest, CheckWebARKitVersion) { // Init the manager with the Akaze tracker manager.initialiseBase(webarkit::TRACKER_TYPE::AKAZE_TRACKER, 640, 480); // Check if the WebARKit version is correct - EXPECT_STREQ(manager.getWebARKitVersion().c_str(), "0.8.0"); + EXPECT_STREQ(manager.getWebARKitVersion().c_str(), "0.9.0"); } // Check cameraProjectionMatrix from manager