Run check-iocs on every module which implements check_indicators() (#903)

A module is part of check-iocs for a platform when it declares the
check-iocs pair, as before. It is now also part of check-iocs when it
overrides check_indicators() and supports at least one command of that
platform.

The rule lives in module_supports_command(). CmdCheckIOCS already uses
that function to pick its modules, so --list-modules and --module follow
it too. Built-in modules are unaffected. check-iocs takes them from
IOS_CHECK_IOCS_MODULES and ANDROID_CHECK_IOCS_MODULES.

A custom module which replaces a built-in module is covered by the same
rule. If it subclasses the module it replaces, it inherits its
check_indicators() and takes over the re-check of the results file. The
"Replacing a built-in module" section of the plugin documentation is
updated to say that.

docs/development/index.md

@@ -78,7 +78,9 @@ MVT logs a warning. The nine pairs are:
 
 `check-iocs` re-checks stored results rather than an acquisition. It matches
 every `<slug>.json` file in the results folder to the module with that slug.
-It then runs that module's `check_indicators()` again.
+It then runs that module's `check_indicators()` again. A module which
+implements `check_indicators()` is included in `check-iocs` for its platform.
+It does not need to declare the `check-iocs` pair.
 
 ### Writing a module
 
@@ -222,12 +224,14 @@ keep running and replace nothing. Every applied substitution is logged, so it
 is recorded in `command.log` when the command runs with an `--output` folder.
 
 Every command resolves replacements on its own. `check-iocs` matches stored
-results files against the slugs of the modules available for that command, so a
-replacement checks the indicators of its own results only if it also declares
-the `("ios", "check-iocs")` pair; otherwise the built-in module it replaced
-re-checks the file. It also matches `--module` on the class name only, so pass
-a differently named replacement's own name there, not the name of the module
-it replaces.
+results files against the slugs of the modules available for that command. A
+replacement which subclasses the module it replaces inherits its
+`check_indicators()`. It is then part of `check-iocs` for its platform and
+re-checks the results file named after its slug. A replacement with no
+`check_indicators()` is not part of `check-iocs`. The built-in module it
+replaced re-checks the file. `check-iocs` matches `--module` on the class name
+only. Pass a differently named replacement's own name there, not the name of
+the module it replaces.
 
 ### Importing from MVT
 

src/mvt/common/module_loader.py

@@ -402,4 +402,14 @@ def module_supports_command(
         )
         return False
 
-    return (platform, command) in {tuple(entry) for entry in supported_commands}
+    pairs = {tuple(entry) for entry in supported_commands}
+    if (platform, command) in pairs:
+        return True
+
+    # A module which implements check_indicators() is re-checked by check-iocs
+    # for its platform. It does not need to declare the check-iocs pair.
+    return (
+        command == "check-iocs"
+        and platform in {entry[0] for entry in pairs if entry}
+        and module_class.check_indicators is not MVTModule.check_indicators
+    )

tests/common/test_cmd_check_iocs.py

@@ -15,6 +15,7 @@ from mvt.common.cmd_check_iocs import CmdCheckIOCS
 from mvt.common.module import MVTModule
 from mvt.ios.cli import cli as ios_cli
 from mvt.ios.command_modules import IOS_CHECK_IOCS_MODULES
+from mvt.ios.modules.backup.manifest import Manifest
 
 # Keep the banner of the group callback from checking for updates online.
 OFFLINE = ["--disable-update-check", "--disable-indicator-update-check"]
@@ -39,34 +40,69 @@ class CustomResultsModule(MVTModule):
         self.checked.append(list(self.results))
 
 
+class BackupCheckerModule(MVTModule):
+    """An iOS module which implements check_indicators() without declaring check-iocs."""
+
+    slug = "backup_checker"
+    supported_commands = (("ios", "check-backup"),)
+
+    checked: list = []
+
+    def run(self) -> None:
+        pass
+
+    def check_indicators(self) -> None:
+        self.checked.append(list(self.results))
+
+
+class BugReportCheckerModule(MVTModule):
+    """The same for Android."""
+
+    slug = "bugreport_checker"
+    supported_commands = (("android", "check-bugreport"),)
+
+    checked: list = []
+
+    def run(self) -> None:
+        pass
+
+    def check_indicators(self) -> None:
+        self.checked.append(list(self.results))
+
+
 class BackupOnlyModule(MVTModule):
-    """A custom module which does not declare check-iocs."""
+    """A custom module which does not implement check_indicators()."""
 
     slug = "backup_only"
     supported_commands = (("ios", "check-backup"),)
 
-    def check_indicators(self) -> None:
-        raise AssertionError("must not be re-checked")
+    def run(self) -> None:
+        pass
 
 
 @pytest.mark.parametrize(
-    "platform, builtin_modules",
-    [("ios", IOS_CHECK_IOCS_MODULES), ("android", ANDROID_CHECK_IOCS_MODULES)],
+    "platform, builtin_modules, checker_module",
+    [
+        ("ios", IOS_CHECK_IOCS_MODULES, BackupCheckerModule),
+        ("android", ANDROID_CHECK_IOCS_MODULES, BugReportCheckerModule),
+    ],
 )
 def test_check_iocs_rechecks_the_stored_results_of_custom_modules(
-    platform, builtin_modules, tmp_path, caplog
+    platform, builtin_modules, checker_module, tmp_path, caplog
 ):
     # check-iocs matches every <slug>.json in the results folder to the module
     # with that slug, custom modules included, and runs its check_indicators()
     # again over the stored results.
     results = [{"domain": "example.org"}]
     (tmp_path / "custom_results.json").write_text(json.dumps(results))
+    (tmp_path / f"{checker_module.slug}.json").write_text(json.dumps(results))
     (tmp_path / "backup_only.json").write_text(json.dumps(results))
     CustomResultsModule.checked.clear()
+    checker_module.checked.clear()
 
     cmd = CmdCheckIOCS(
         target_path=str(tmp_path),
-        custom_modules=[CustomResultsModule, BackupOnlyModule],
+        custom_modules=[CustomResultsModule, checker_module, BackupOnlyModule],
         platform=platform,
     )
     cmd.modules = builtin_modules
@@ -74,29 +110,79 @@ def test_check_iocs_rechecks_the_stored_results_of_custom_modules(
     with caplog.at_level(logging.INFO):
         cmd.run()
 
+    # A module which declares the check-iocs pair is re-checked.
     assert CustomResultsModule.checked == [results]
     assert (
         'Loading results from "custom_results.json" with module CustomResultsModule'
         in caplog.text
     )
-    # A module declaring only check-backup is not part of check-iocs.
+    # So is a module which only implements check_indicators().
+    assert checker_module.checked == [results]
+    # A module which does neither is not part of check-iocs.
     assert "backup_only.json" not in caplog.text
 
 
-def test_check_iocs_lists_custom_modules_declaring_the_command(caplog):
+@pytest.mark.parametrize(
+    "platform, builtin_modules, listed, not_listed",
+    [
+        (
+            "ios",
+            IOS_CHECK_IOCS_MODULES,
+            "BackupCheckerModule",
+            "BugReportCheckerModule",
+        ),
+        (
+            "android",
+            ANDROID_CHECK_IOCS_MODULES,
+            "BugReportCheckerModule",
+            "BackupCheckerModule",
+        ),
+    ],
+)
+def test_check_iocs_lists_the_custom_modules_it_runs(
+    platform, builtin_modules, listed, not_listed, caplog
+):
     cmd = CmdCheckIOCS(
-        custom_modules=[CustomResultsModule, BackupOnlyModule],
-        platform="ios",
+        custom_modules=[
+            CustomResultsModule,
+            BackupCheckerModule,
+            BugReportCheckerModule,
+            BackupOnlyModule,
+        ],
+        platform=platform,
     )
-    cmd.modules = IOS_CHECK_IOCS_MODULES
+    cmd.modules = builtin_modules
 
     with caplog.at_level(logging.INFO):
         cmd.list_modules()
 
     assert "CustomResultsModule" in caplog.text
+    # The module which implements check_indicators() for this platform is listed.
+    assert listed in caplog.text
+    # The one for the other platform is not, and neither is BackupOnlyModule.
+    assert not_listed not in caplog.text
     assert "BackupOnlyModule" not in caplog.text
 
 
+class ReplacementManifest(Manifest):
+    """A replacement for a built-in module which does not declare check-iocs."""
+
+    supported_commands = (("ios", "check-backup"),)
+    replaces = Manifest
+
+
+def test_check_iocs_uses_a_replacement_of_a_built_in_module():
+    # A replacement which subclasses a built-in module inherits its
+    # check_indicators(). check-iocs then runs it in place of that module.
+    cmd = CmdCheckIOCS(custom_modules=[ReplacementManifest], platform="ios")
+    cmd.modules = IOS_CHECK_IOCS_MODULES
+
+    available = cmd._available_modules()
+
+    assert ReplacementManifest in available
+    assert Manifest not in available
+
+
 LOADED_MODULE = '''
 from mvt.common.module import MVTModule