File bugs in Core :: General when the moz.yaml's component doesn't exist

apis/bugzilla_api.py

@@ -49,7 +49,7 @@ def _load_json_or_raise(r, where: str):
         ) from e
 
 
-def fileBug(url, apikey, ff_version, product, component, summary, description, cc_list, needinfo, see_also, depends_on, blocks, moco_confidential):
+def fileBug(url, apikey, ff_version, product, component, summary, description, cc_list, needinfo, see_also, depends_on, blocks, moco_confidential, mark_affected=True):
     assert isinstance(cc_list, list)
 
     data = {
@@ -65,7 +65,7 @@ def fileBug(url, apikey, ff_version, product, component, summary, description, c
         'cc': ['tom@mozilla.com', 'jewilde@mozilla.com'] + cc_list
     }
 
-    if is_prod(url):
+    if is_prod(url) and mark_affected:
         data['cf_status_firefox' + str(ff_version)] = 'affected'
 
     if see_also:

components/bugzilla.py

@@ -198,6 +198,15 @@ def COULD_NOT_VENDOR_ALL_FILES(library, errormessage):
         s += "they are excluded from the source tree and ignored in the future."
         return s
 
+    @staticmethod
+    def MISSING_COMPONENT(library, filed_bug_id):
+        return ("%s specifies the Bugzilla component '%s :: %s', which doesn't exist, so Updatebot "
+                "filed bug %s in %s :: %s instead.\n\n"
+                "Please update the bugzilla product and component in %s, and move bug %s to the right component.") % (
+            library.yaml_path, library.bugzilla_product, library.bugzilla_component,
+            filed_bug_id, FALLBACK_PRODUCT_COMPONENT[0], FALLBACK_PRODUCT_COMPONENT[1],
+            library.yaml_path, filed_bug_id)
+
     @staticmethod
     def TRY_RUN_SUBMITTED(revision, another=False):
         return "I've submitted a" + ("nother" if another else "") + " try run for this commit: https://treeherder.mozilla.org/jobs?repo=try&revision=" + revision
@@ -214,6 +223,21 @@ def is_needinfo_exception(e):
     return 'is not currently accepting "needinfo" requests' in str(e)
 
 
+def is_missing_component_exception(e):
+    # fileBug raises Exception(<Bugzilla's JSON error>); code 51 is "object
+    # does not exist", which also covers e.g. unknown users.
+    j = e.args[0] if e.args else None
+    return isinstance(j, dict) and j.get('code') == 51 and \
+        'There is no component named' in j.get('message', '')
+
+
+# Where a bug goes when the component in its moz.yaml doesn't exist, and where
+# the follow-up bug asking for the moz.yaml to be fixed goes.
+FALLBACK_PRODUCT_COMPONENT = ("Core", "General")
+UPDATEBOT_PRODUCT_COMPONENT = ("Developer Infrastructure", "Mach Vendor & Updatebot")
+UPDATEBOT_OWNER_BZ = "tom@mozilla.com"
+
+
 class BugzillaProvider(BaseProvider, INeedsLoggingProvider):
     PROTECTED_LANDING_LINK_PREFIXES = (
         "https://hg.mozilla.org/integration/autoland/",
@@ -240,19 +264,46 @@ def file_bug(self, library, summary, description, cc_list, needinfo=None, see_al
             description = description[0:65533 - len(suffix)] + suffix
 
         try:
-            bugID = fileBug(self.config['url'], self.config['apikey'], self.config['General']['ff-version'],
-                            library.bugzilla_product, library.bugzilla_component,
-                            summary, description, cc_list, needinfo, see_also, depends_on, blocks, moco_confidential)
+            bugID = self._file_bug(library.bugzilla_product, library.bugzilla_component,
+                                   summary, description, cc_list, needinfo, see_also, depends_on, blocks, moco_confidential)
         except Exception as e:
-            if is_needinfo_exception(e):
-                self.logger.log("Developer is not accepting needinfos, trying again without...", level=LogLevel.Info)
-                return self.file_bug(library, summary, description, cc_list, None, see_also, depends_on, blocks, moco_confidential)
-            else:
+            if not is_missing_component_exception(e):
                 raise e
+            self.logger.log("The component %s :: %s from %s doesn't exist; filing in %s :: %s instead." % (
+                library.bugzilla_product, library.bugzilla_component, library.yaml_path,
+                FALLBACK_PRODUCT_COMPONENT[0], FALLBACK_PRODUCT_COMPONENT[1]), level=LogLevel.Warning)
+            bugID = self._file_bug(FALLBACK_PRODUCT_COMPONENT[0], FALLBACK_PRODUCT_COMPONENT[1],
+                                   summary, description, cc_list, needinfo, see_also, depends_on, blocks, moco_confidential)
+            self._file_missing_component_bug(library, bugID)
 
         self.logger.log("Filed Bug with ID", bugID, level=LogLevel.Info)
         return bugID
 
+    def _file_bug(self, product, component, summary, description, cc_list, needinfo, see_also, depends_on, blocks, moco_confidential, mark_affected=True):
+        try:
+            return fileBug(self.config['url'], self.config['apikey'], self.config['General']['ff-version'],
+                           product, component, summary, description, cc_list, needinfo, see_also, depends_on, blocks, moco_confidential,
+                           mark_affected=mark_affected)
+        except Exception as e:
+            if is_needinfo_exception(e):
+                self.logger.log("Developer is not accepting needinfos, trying again without...", level=LogLevel.Info)
+                return self._file_bug(product, component, summary, description, cc_list, None, see_also, depends_on, blocks, moco_confidential, mark_affected)
+            raise e
+
+    def _file_missing_component_bug(self, library, filed_bug_id):
+        needinfo = [UPDATEBOT_OWNER_BZ]
+        if library.maintainer_bz and library.maintainer_bz != UPDATEBOT_OWNER_BZ:
+            needinfo.append(library.maintainer_bz)
+        try:
+            bugID = self._file_bug(UPDATEBOT_PRODUCT_COMPONENT[0], UPDATEBOT_PRODUCT_COMPONENT[1],
+                                   "Update the Bugzilla component in %s" % library.yaml_path,
+                                   CommentTemplates.MISSING_COMPONENT(library, filed_bug_id),
+                                   [], needinfo, None, None, filed_bug_id, False, mark_affected=False)
+            self.logger.log("Filed Bug %s about the missing component, blocking bug %s" % (bugID, filed_bug_id), level=LogLevel.Info)
+        except Exception as e:
+            # The library's bug is already filed; don't lose track of it over this.
+            self.logger.log("Could not file a bug about the missing component in %s: %s" % (library.yaml_path, e), level=LogLevel.Error)
+
     @logEntryExit
     def comment_on_bug(self, bug_id, comment, needinfo=None, assignee=None):
         if len(comment) > 65535:

tests/bugzilla.py

@@ -25,6 +25,7 @@
 
 class MockBugzillaServer(server.BaseHTTPRequestHandler):
     closed_bug_ids = []
+    filed_bugs = []
 
     def do_POST(self):
         expectedPath_file = "/bug?api_key=bob"
@@ -35,7 +36,17 @@ def do_POST(self):
         self.send_header("Content-type", "application/json")
         self.end_headers()
 
-        if expectedPath_file == self.path:
+        MockBugzillaServer.filed_bugs.append(content)
+        if content['component'] == 'Missing Component':
+            self.wfile.write(json.dumps({'error': True, 'code': 51, 'message': "There is no component named 'Missing Component' in the 'Core' product."}).encode())
+        elif content['product'] == 'Core' and content['component'] == 'General':
+            self.wfile.write("{\"id\":457}".encode())
+        elif content['component'] == 'Mach Vendor & Updatebot':
+            if 'broken' in content['summary']:
+                self.wfile.write(json.dumps({'error': True, 'code': 100, 'message': "Something went wrong."}).encode())
+            else:
+                self.wfile.write("{\"id\":458}".encode())
+        elif expectedPath_file == self.path:
             expectedContent = {
                 'version': "unspecified",
                 'op_sys': "unspecified",
@@ -194,6 +205,38 @@ def testComment(self):
         self.bugzillaProvider.comment_on_bug(
             1237, "X" * 70000, assignee='Jon')
 
+    def testFileMissingComponent(self):
+        library = Struct(**{
+            'name': 'mylib',
+            'bugzilla_product': 'Core',
+            'bugzilla_component': 'Missing Component',
+            'yaml_path': 'third_party/mylib/moz.yaml',
+            'maintainer_bz': 'owner@example.com',
+        })
+        bug_id = self.bugzillaProvider.file_bug(library, "Update mylib to new version V1", "Details", [])
+        self.assertEqual(bug_id, 457, "The bug should have been filed in Core :: General")
+
+        filed = [b for b in MockBugzillaServer.filed_bugs if 'mylib' in b['summary']]
+        self.assertEqual([(b['product'], b['component']) for b in filed],
+                         [('Core', 'Missing Component'), ('Core', 'General'), ('Developer Infrastructure', 'Mach Vendor & Updatebot')])
+        library_bug, followup = filed[1], filed[2]
+        self.assertIn('cf_status_firefox88', library_bug)
+        self.assertEqual(followup['blocks'], 457)
+        self.assertEqual([f['requestee'] for f in followup['flags']], ['tom@mozilla.com', 'owner@example.com'])
+        self.assertNotIn('cf_status_firefox88', followup)
+        self.assertIn('third_party/mylib/moz.yaml', followup['description'])
+
+    def testFileMissingComponentFollowUpFails(self):
+        library = Struct(**{
+            'name': 'broken',
+            'bugzilla_product': 'Core',
+            'bugzilla_component': 'Missing Component',
+            'yaml_path': 'third_party/broken/moz.yaml',
+            'maintainer_bz': 'owner@example.com',
+        })
+        # Failing to file the follow-up must not lose the library's bug.
+        self.assertEqual(self.bugzillaProvider.file_bug(library, "Update broken to new version V1", "Details", []), 457)
+
     def testStatus(self):
         self.bugzillaProvider.mark_ff_version_affected(456, 76)