-
-
Notifications
You must be signed in to change notification settings - Fork 265
Add V2 Importer for Tuxcare advisories #2104
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
0bb18b5
8039472
bce8139
ea5bae5
83bb44d
d464ef9
0a8456a
663d5fd
640e80f
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,107 @@ | ||
| import json | ||
| import logging | ||
| from typing import Iterable | ||
|
|
||
| from dateutil.parser import parse | ||
| from packageurl import PackageURL | ||
| from pytz import UTC | ||
| from univers.version_range import GenericVersionRange | ||
|
|
||
| from vulnerabilities.importer import AdvisoryData | ||
| from vulnerabilities.importer import AffectedPackageV2 | ||
| from vulnerabilities.importer import VulnerabilitySeverity | ||
| from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 | ||
| from vulnerabilities.severity_systems import GENERIC | ||
| from vulnerabilities.utils import fetch_response | ||
|
|
||
| logger = logging.getLogger(__name__) | ||
|
|
||
|
|
||
| class TuxCareImporterPipeline(VulnerableCodeBaseImporterPipelineV2): | ||
| pipeline_id = "tuxcare_importer_v2" | ||
| spdx_license_expression = "Apache-2.0" | ||
| license_url = "https://tuxcare.com/legal" | ||
|
|
||
| @classmethod | ||
| def steps(cls): | ||
| return ( | ||
| cls.fetch, | ||
| cls.collect_and_store_advisories, | ||
| ) | ||
|
|
||
| def fetch(self) -> None: | ||
| url = "https://cve.tuxcare.com/els/download-json?orderBy=updated-desc" | ||
| self.log(f"Fetching `{url}`") | ||
| response = fetch_response(url) | ||
| self.response = response.json() if response else [] | ||
|
|
||
| def advisories_count(self) -> int: | ||
| return len(self.response) | ||
|
|
||
| def _create_purl(self, project_name: str, os_name: str) -> PackageURL: | ||
| qualifiers = {} | ||
| if os_name: | ||
| qualifiers["distro"] = os_name | ||
|
|
||
| return PackageURL( | ||
| type="generic", namespace="tuxcare", name=project_name, qualifiers=qualifiers | ||
| ) | ||
|
|
||
| def collect_advisories(self) -> Iterable[AdvisoryData]: | ||
| for record in self.response: | ||
| cve_id = record.get("cve", "").strip() | ||
| if not cve_id or not cve_id.startswith("CVE-"): | ||
| continue | ||
|
Comment on lines
+53
to
+54
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Please avoid silently skipping this |
||
|
|
||
| os_name = record.get("os_name", "").strip() | ||
| project_name = record.get("project_name", "").strip() | ||
| version = record.get("version", "").strip() | ||
| score = record.get("score", "").strip() | ||
| severity = record.get("severity", "").strip() | ||
| last_updated = record.get("last_updated", "").strip() | ||
|
|
||
| advisory_id = cve_id | ||
|
|
||
| affected_packages = [] | ||
| if project_name: | ||
| purl = self._create_purl(project_name, os_name) | ||
|
|
||
| affected_version_range = None | ||
| if version: | ||
| try: | ||
| affected_version_range = GenericVersionRange.from_versions([version]) | ||
| except ValueError as e: | ||
| logger.warning(f"Failed to parse version {version} for {cve_id}: {e}") | ||
|
Comment on lines
+69
to
+74
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why are we assuming that all versions are affected? According to my understanding, different versions can have different statuses: |
||
|
|
||
| affected_packages.append( | ||
| AffectedPackageV2( | ||
| package=purl, | ||
| affected_version_range=affected_version_range, | ||
| ) | ||
| ) | ||
|
|
||
| severities = [] | ||
| if severity and score: | ||
| severities.append( | ||
| VulnerabilitySeverity( | ||
| system=GENERIC, | ||
| value=score, | ||
| scoring_elements=severity, | ||
| ) | ||
| ) | ||
|
|
||
| date_published = None | ||
| if last_updated: | ||
| try: | ||
| date_published = parse(last_updated).replace(tzinfo=UTC) | ||
| except ValueError as e: | ||
| logger.warning(f"Failed to parse date {last_updated} for {cve_id}: {e}") | ||
|
|
||
| yield AdvisoryData( | ||
| advisory_id=advisory_id, | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Advisory_id should be unique, so drop the old database and run the importer again to ensure there is no duplication
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Hey @ziadhany! I've analyzed the data and found duplicate advisory IDs in the database. This happens because TuxCare structures their data with the same CVE ID across different OS distributions, packages, and versions - each record represents the same CVE but for a specific OS+package+version combination. For example, CVE-2023-52922 appears in 17 different records: {
"cve": "CVE-2023-52922",
"os_name": "CloudLinux 7 ELS",
"project_name": "squid",
"version": "3.5.20",
"score": "7.8",
"severity": "HIGH",
"status": "In Testing",
"last_updated": "2025-12-23 10:08:36.423446"
}
{
"cve": "CVE-2023-52922",
"os_name": "Oracle Linux 7 ELS",
"project_name": "squid",
"version": "3.5.20",
"score": "7.8",
"severity": "HIGH",
"status": "In Testing",
"last_updated": "2025-12-23 10:08:35.944749"
}
{
"cve": "CVE-2023-52922",
"os_name": "CentOS 8.5 ELS",
"project_name": "kernel",
"version": "4.18.0",
"score": "7.8",
"severity": "HIGH",
"status": "Released",
"last_updated": "2025-05-21 01:43:28.677045"
}Workaround This way each OS+package+version combination gets a unique advisory, the CVE remains searchable via aliases, and we avoid duplicates. Does this approach work for you? Do share your suggestions. Also, just a heads up - the TuxCare endpoint is currently returning a 500 error, it might take some time before I can run the importer and verify everything works as expected. |
||
| affected_packages=affected_packages, | ||
| severities=severities, | ||
| date_published=date_published, | ||
| url=f"https://cve.tuxcare.com/els/cve/{cve_id}", | ||
| original_advisory_text=json.dumps(record, indent=2, ensure_ascii=False), | ||
| ) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| # | ||
| # Copyright (c) nexB Inc. and others. All rights reserved. | ||
| # VulnerableCode is a trademark of nexB Inc. | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
| # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. | ||
| # See https://github.com/aboutcode-org/vulnerablecode for support or download. | ||
| # See https://aboutcode.org for more information about nexB OSS projects. | ||
| # | ||
|
|
||
| import json | ||
| from pathlib import Path | ||
| from unittest import TestCase | ||
| from unittest.mock import Mock | ||
| from unittest.mock import patch | ||
|
|
||
| from vulnerabilities.pipelines.v2_importers.tuxcare_importer import TuxCareImporterPipeline | ||
| from vulnerabilities.tests import util_tests | ||
|
|
||
| TEST_DATA = Path(__file__).parent.parent.parent / "test_data" / "tuxcare" | ||
|
|
||
|
|
||
| class TestTuxCareImporterPipeline(TestCase): | ||
| @patch("vulnerabilities.pipelines.v2_importers.tuxcare_importer.fetch_response") | ||
| def test_collect_advisories(self, mock_fetch): | ||
| sample_path = TEST_DATA / "data.json" | ||
| sample_data = json.loads(sample_path.read_text(encoding="utf-8")) | ||
|
|
||
| mock_fetch.return_value = Mock(json=lambda: sample_data) | ||
|
|
||
| pipeline = TuxCareImporterPipeline() | ||
| pipeline.fetch() | ||
|
|
||
| advisories = [data.to_dict() for data in list(pipeline.collect_advisories())] | ||
|
|
||
| expected_file = TEST_DATA / "expected.json" | ||
| util_tests.check_results_against_json(advisories, expected_file) | ||
|
|
||
| assert pipeline.advisories_count() == 5 |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| [ | ||
| { | ||
| "cve": "CVE-2023-52922", | ||
| "os_name": "CloudLinux 7 ELS", | ||
| "project_name": "squid", | ||
| "version": "3.5.20", | ||
| "score": "7.8", | ||
| "severity": "HIGH", | ||
| "status": "In Testing", | ||
| "last_updated": "2025-12-23 10:08:36.423446" | ||
| }, | ||
| { | ||
| "cve": "CVE-2023-52922", | ||
| "os_name": "Oracle Linux 7 ELS", | ||
| "project_name": "squid", | ||
| "version": "3.5.20", | ||
| "score": "7.8", | ||
| "severity": "HIGH", | ||
| "status": "In Testing", | ||
| "last_updated": "2025-12-23 10:08:35.944749" | ||
| }, | ||
| { | ||
| "cve": "CVE-2023-48161", | ||
| "os_name": "RHEL 7 ELS", | ||
| "project_name": "java-11-openjdk", | ||
| "version": "11.0.23", | ||
| "score": "7.1", | ||
| "severity": "HIGH", | ||
| "status": "In Progress", | ||
| "last_updated": "2025-12-23 08:55:12.096092" | ||
| }, | ||
| { | ||
| "cve": "CVE-2024-21147", | ||
| "os_name": "RHEL 7 ELS", | ||
| "project_name": "java-11-openjdk", | ||
| "version": "11.0.23", | ||
| "score": "7.4", | ||
| "severity": "HIGH", | ||
| "status": "In Progress", | ||
| "last_updated": "2025-12-23 08:55:07.139188" | ||
| }, | ||
| { | ||
| "cve": "CVE-2025-21587", | ||
| "os_name": "RHEL 7 ELS", | ||
| "project_name": "java-11-openjdk", | ||
| "version": "11.0.23", | ||
| "score": "7.4", | ||
| "severity": "HIGH", | ||
| "status": "In Progress", | ||
| "last_updated": "2025-12-23 08:55:06.706873" | ||
| } | ||
| ] |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,102 @@ | ||
| [ | ||
| { | ||
| "advisory_id": "CVE-2023-52922", | ||
| "aliases": [], | ||
| "summary": "", | ||
| "affected_packages": [ | ||
| { | ||
| "package": {"type": "generic", "namespace": "tuxcare", "name": "squid", "version": "", "qualifiers": "distro=CloudLinux%207%20ELS", "subpath": ""}, | ||
| "affected_version_range": "vers:generic/3.5.20", | ||
| "fixed_version_range": null, | ||
| "introduced_by_commit_patches": [], | ||
| "fixed_by_commit_patches": [] | ||
| } | ||
| ], | ||
| "references_v2": [], | ||
| "patches": [], | ||
| "severities": [{"system": "generic_textual", "value": "7.8", "scoring_elements": "HIGH"}], | ||
| "date_published": "2025-12-23T10:08:36.423446+00:00", | ||
| "weaknesses": [], | ||
| "url": "https://cve.tuxcare.com/els/cve/CVE-2023-52922" | ||
| }, | ||
| { | ||
| "advisory_id": "CVE-2023-52922", | ||
| "aliases": [], | ||
| "summary": "", | ||
| "affected_packages": [ | ||
| { | ||
| "package": {"type": "generic", "namespace": "tuxcare", "name": "squid", "version": "", "qualifiers": "distro=Oracle%20Linux%207%20ELS", "subpath": ""}, | ||
| "affected_version_range": "vers:generic/3.5.20", | ||
| "fixed_version_range": null, | ||
| "introduced_by_commit_patches": [], | ||
| "fixed_by_commit_patches": [] | ||
| } | ||
| ], | ||
| "references_v2": [], | ||
| "patches": [], | ||
| "severities": [{"system": "generic_textual", "value": "7.8", "scoring_elements": "HIGH"}], | ||
| "date_published": "2025-12-23T10:08:35.944749+00:00", | ||
| "weaknesses": [], | ||
| "url": "https://cve.tuxcare.com/els/cve/CVE-2023-52922" | ||
| }, | ||
| { | ||
| "advisory_id": "CVE-2023-48161", | ||
| "aliases": [], | ||
| "summary": "", | ||
| "affected_packages": [ | ||
| { | ||
| "package": {"type": "generic", "namespace": "tuxcare", "name": "java-11-openjdk", "version": "", "qualifiers": "distro=RHEL%207%20ELS", "subpath": ""}, | ||
| "affected_version_range": "vers:generic/11.0.23", | ||
| "fixed_version_range": null, | ||
| "introduced_by_commit_patches": [], | ||
| "fixed_by_commit_patches": [] | ||
| } | ||
| ], | ||
| "references_v2": [], | ||
| "patches": [], | ||
| "severities": [{"system": "generic_textual", "value": "7.1", "scoring_elements": "HIGH"}], | ||
| "date_published": "2025-12-23T08:55:12.096092+00:00", | ||
| "weaknesses": [], | ||
| "url": "https://cve.tuxcare.com/els/cve/CVE-2023-48161" | ||
| }, | ||
| { | ||
| "advisory_id": "CVE-2024-21147", | ||
| "aliases": [], | ||
| "summary": "", | ||
| "affected_packages": [ | ||
| { | ||
| "package": {"type": "generic", "namespace": "tuxcare", "name": "java-11-openjdk", "version": "", "qualifiers": "distro=RHEL%207%20ELS", "subpath": ""}, | ||
| "affected_version_range": "vers:generic/11.0.23", | ||
| "fixed_version_range": null, | ||
| "introduced_by_commit_patches": [], | ||
| "fixed_by_commit_patches": [] | ||
| } | ||
| ], | ||
| "references_v2": [], | ||
| "patches": [], | ||
| "severities": [{"system": "generic_textual", "value": "7.4", "scoring_elements": "HIGH"}], | ||
| "date_published": "2025-12-23T08:55:07.139188+00:00", | ||
| "weaknesses": [], | ||
| "url": "https://cve.tuxcare.com/els/cve/CVE-2024-21147" | ||
| }, | ||
| { | ||
| "advisory_id": "CVE-2025-21587", | ||
| "aliases": [], | ||
| "summary": "", | ||
| "affected_packages": [ | ||
| { | ||
| "package": {"type": "generic", "namespace": "tuxcare", "name": "java-11-openjdk", "version": "", "qualifiers": "distro=RHEL%207%20ELS", "subpath": ""}, | ||
| "affected_version_range": "vers:generic/11.0.23", | ||
| "fixed_version_range": null, | ||
| "introduced_by_commit_patches": [], | ||
| "fixed_by_commit_patches": [] | ||
| } | ||
| ], | ||
| "references_v2": [], | ||
| "patches": [], | ||
| "severities": [{"system": "generic_textual", "value": "7.4", "scoring_elements": "HIGH"}], | ||
| "date_published": "2025-12-23T08:55:06.706873+00:00", | ||
| "weaknesses": [], | ||
| "url": "https://cve.tuxcare.com/els/cve/CVE-2025-21587" | ||
| } | ||
| ] |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@Samk1710 I think this purl is incorrect, sorry for the confusion.
I noticed that TuxCare provides security patches for Endless Lifecycle Support (ELS), delivering qualified security and selected bug-fix errata advisories across all architectures.
So, I think the correct way to do this is that we should create a mapping.
For unknown OS, use generic.