Update match to matchcode (#1365)

* Use correct function in poll_matching_results

Signed-off-by: Jono Yang <jyang@nexb.com>

* Only check for success or failure in poll_until_success

Signed-off-by: Jono Yang <jyang@nexb.com>

* Properly collect match results in MatchToMatchCode

Signed-off-by: Jono Yang <jyang@nexb.com>

* Update CHANGELOG.rst

Signed-off-by: Jono Yang <jyang@nexb.com>

---------

Signed-off-by: Jono Yang <jyang@nexb.com>

Author: JonoYang

CHANGELOG.rst

@@ -51,6 +51,9 @@ v34.8.0 (2024-08-15)
   a project.
   https://github.com/nexB/scancode.io/issues/1182
 
+- Fix issues in ``match_to_matchcode`` where the incorrect polling function was
+  used and match results were not properly collected.
+
 v34.7.1 (2024-07-15)
 --------------------
 

scanpipe/pipelines/match_to_matchcode.py

@@ -70,13 +70,15 @@ def check_matchcode_service_availability(self):
 
     def send_project_json_to_matchcode(self):
         """Create a JSON scan of the project Codebase and send it to MatchCode.io."""
-        self.run_url = matchcode.send_project_json_to_matchcode(self.project)
+        self.match_url, self.run_url = matchcode.send_project_json_to_matchcode(
+            self.project
+        )
 
     def poll_matching_results(self):
         """Wait until the match results are ready by polling the match run status."""
-        matchcode.poll_until_success(self.run_url)
+        matchcode.poll_run_url_status(self.run_url)
 
     def create_packages_from_match_results(self):
         """Create DiscoveredPackages from match results."""
-        match_results = matchcode.get_match_results(self.run_url)
+        match_results = matchcode.get_match_results(self.match_url)
         matchcode.create_packages_from_match_results(self.project, match_results)

scanpipe/pipes/__init__.py

@@ -442,27 +442,18 @@ def poll_until_success(check, sleep=10, **kwargs):
     function.
     """
     run_status = AbstractTaskFieldsModel.Status
-    # Continue looping if the run instance has the following statuses
-    CONTINUE_STATUSES = [
-        run_status.NOT_STARTED,
-        run_status.QUEUED,
-        run_status.RUNNING,
-    ]
     # Return False if the run instance has the following statuses
     FAIL_STATUSES = [
         run_status.FAILURE,
         run_status.STOPPED,
         run_status.STALE,
     ]
-
     while True:
         status = check(**kwargs)
+
         if status == run_status.SUCCESS:
             return True
 
-        if status in CONTINUE_STATUSES:
-            continue
-
         if status in FAIL_STATUSES:
             return False
 

scanpipe/pipes/matchcode.py

@@ -266,8 +266,9 @@ def send_project_json_to_matchcode(
             timeout=timeout,
             files=files,
         )
+    match_url = response["url"]
     run_url = response["runs"][0]["url"]
-    return run_url
+    return match_url, run_url
 
 
 def get_run_url_status(run_url, **kwargs):
@@ -299,19 +300,26 @@ def poll_run_url_status(run_url, sleep=10):
         raise MatchCodeIOException(msg)
 
 
-def get_match_results(run_url):
+def create_match_results_url(match_url):
     """
-    Given the `run_url` for a pipeline running the matchcode matching pipeline,
-    return the match results for that run.
+    Given the `match_url` for a project running the matchcode matching pipeline,
+    return the match results URL from `match_url`.
     """
-    response = request_get(run_url)
-    project_url = response["project"]
     # `project_url` can have params, such as "?format=json"
-    if "?" in project_url:
-        project_url, _ = project_url.split("?")
-    project_url = project_url.rstrip("/")
-    results_url = project_url + "/results/"
-    return request_get(results_url)
+    if "?" in match_url:
+        match_url, _ = match_url.split("?")
+    match_url = match_url.rstrip("/")
+    match_url = match_url + "/results/"
+    return match_url
+
+
+def get_match_results(match_url):
+    """
+    Given the `match_url` for a project running the matchcode matching pipeline,
+    return the match results.
+    """
+    match_results_url = create_match_results_url(match_url)
+    return request_get(match_results_url)
 
 
 def map_match_results(match_results):

scanpipe/tests/pipes/test_matchcode.py

@@ -76,10 +76,14 @@ def mock_request_post_return(url, files, timeout):
 
         mock_request_post.side_effect = mock_request_post_return
 
-        run_url = matchcode.send_project_json_to_matchcode(self.project1)
+        match_url, run_url = matchcode.send_project_json_to_matchcode(self.project1)
+        expected_match_url = (
+            "http://192.168.1.12/api/matching/65bf1e6d-6bff-4841-9c9b-db5cf25edfa7/"
+        )
         expected_run_url = (
             "http://192.168.1.12/api/runs/52b2930d-6e85-4b3e-ba3e-17dd9a618650/"
         )
+        self.assertEqual(expected_match_url, match_url)
         self.assertEqual(expected_run_url, run_url)
 
     @mock.patch("scanpipe.pipes.matchcode.request_get")
@@ -280,22 +284,21 @@ def test_scanpipe_pipes_matchcode_create_packages_from_match_results(self):
         # This resource should not have a Package match
         self.assertFalse(0, len(r2.for_packages))
 
+    def test_scanpipe_pipes_matchcode_create_match_results_url(self):
+        match_url = (
+            "http://192.168.1.12/api/matching/65bf1e6d-6bff-4841-9c9b-db5cf25edfa7/"
+        )
+        expected_match_url = "http://192.168.1.12/api/matching/65bf1e6d-6bff-4841-9c9b-db5cf25edfa7/results/"
+        self.assertEqual(
+            expected_match_url, matchcode.create_match_results_url(match_url)
+        )
+
     @mock.patch("scanpipe.pipes.matchcode.request_get")
     @mock.patch("scanpipe.pipes.matchcode.is_available")
     def test_scanpipe_pipes_matchcode_get_match_results(
         self, mock_is_available, mock_request_get
     ):
         mock_is_available.return_value = True
-
-        request_get_check_response_loc = (
-            self.data
-            / "matchcode"
-            / "match_to_matchcode"
-            / "request_get_check_response.json"
-        )
-        with open(request_get_check_response_loc) as f:
-            mock_request_get_check_return = json.load(f)
-
         request_get_results_response_loc = (
             self.data
             / "matchcode"
@@ -305,13 +308,13 @@ def test_scanpipe_pipes_matchcode_get_match_results(
         with open(request_get_results_response_loc) as f:
             mock_request_get_results_return = json.load(f)
         mock_request_get.side_effect = [
-            mock_request_get_check_return,
             mock_request_get_results_return,
         ]
 
-        run_url = "http://192.168.1.12/api/runs/52b2930d-6e85-4b3e-ba3e-17dd9a618650/"
-        match_results = matchcode.get_match_results(run_url)
-
+        match_url = (
+            "http://192.168.1.12/api/matching/65bf1e6d-6bff-4841-9c9b-db5cf25edfa7/"
+        )
+        match_results = matchcode.get_match_results(match_url)
         self.assertEqual(mock_request_get_results_return, match_results)
 
     def test_scanpipe_pipes_matchcode_fingerprint_codebase_resources(self):