Add npm dist gen supporting for multi-targets and integrity computing

  * Add registry in target bucket configuration
  * Different registries dist-tarball gen for multi-targets
  * Tarball integrity computing for dist support
  * Add unit testings for dist gen computing against s3

Author: yma955

charon/cmd/command.py

@@ -345,14 +345,15 @@ def delete(
             __safe_delete(tmp_dir)
 
 
-def __get_targets(target: List[str], conf: CharonConfig) -> List[Tuple[str, str, str]]:
+def __get_targets(target: List[str], conf: CharonConfig) -> List[Tuple[str, str, str, str]]:
     targets_ = []
     for tgt in target:
         aws_bucket = conf.get_aws_bucket(tgt)
         if not aws_bucket:
             continue
         prefix = conf.get_bucket_prefix(tgt)
-        targets_.append([tgt, aws_bucket, prefix])
+        registry = conf.get_bucket_registry(tgt)
+        targets_.append([tgt, aws_bucket, prefix, registry])
     if len(targets_) == 0:
         logger.error(
             "All targets are not valid or configured, "

charon/config.py

@@ -20,6 +20,7 @@
 import logging
 
 from charon.utils.strings import remove_prefix
+from charon.constants import DEFAULT_REGISTRY
 
 CONFIG_FILE = "charon.yaml"
 
@@ -53,8 +54,8 @@ def get_aws_bucket(self, target: str) -> str:
             return None
         bucket = target_.get("bucket", None)
         if not bucket:
-            logger.error("The bucket %s is not found for target %s "
-                         "in charon configuration.")
+            logger.error("The bucket is not found for target %s "
+                         "in charon configuration.", target)
         return bucket
 
     def get_bucket_prefix(self, target: str) -> str:
@@ -73,6 +74,19 @@ def get_bucket_prefix(self, target: str) -> str:
         prefix = remove_prefix(prefix, "/")
         return prefix
 
+    def get_bucket_registry(self, target: str) -> str:
+        target_: Dict = self.__targets.get(target, None)
+        if not target_ or not isinstance(target_, Dict):
+            logger.error("The target %s is not found in charon configuration.", target)
+            return None
+        registry = target_.get("registry", None)
+        if not registry:
+            registry = DEFAULT_REGISTRY
+            logger.error("The registry is not found for target %s "
+                         "in charon configuration, so DEFAULT_REGISTRY(localhost) will be used.",
+                         target)
+        return registry
+
     def get_manifest_bucket(self) -> str:
         return self.__manifest_bucket
 

charon/constants.py

@@ -174,4 +174,4 @@
 MANIFEST_SUFFIX = ".txt"
 DEFAULT_ERRORS_LOG = "errors.log"
 
-NRRC_REGISTRY = "npm.registry.redhat.com"
+DEFAULT_REGISTRY = "localhost"

charon/pkgs/maven.py

@@ -256,7 +256,7 @@ def handle_maven_uploading(
     prod_key: str,
     ignore_patterns=None,
     root="maven-repository",
-    targets: List[Tuple[str, str, str]] = None,
+    targets: List[Tuple[str, str, str, str]] = None,
     aws_profile=None,
     dir_=None,
     do_index=True,
@@ -418,7 +418,7 @@ def handle_maven_del(
     prod_key: str,
     ignore_patterns=None,
     root="maven-repository",
-    targets: List[Tuple[str, str, str]] = None,
+    targets: List[Tuple[str, str, str, str]] = None,
     aws_profile=None,
     dir_=None,
     do_index=True,

charon/pkgs/npm.py

@@ -30,6 +30,7 @@
 from charon.pkgs.pkg_utils import upload_post_process, rollback_post_process
 from charon.utils.strings import remove_prefix
 from charon.utils.files import write_manifest
+from charon.utils.map import del_none
 
 logger = logging.getLogger(__name__)
 
@@ -65,7 +66,7 @@ def __init__(self, metadata, is_version):
 def handle_npm_uploading(
         tarball_path: str,
         product: str,
-        targets: List[Tuple[str, str, str]] = None,
+        targets: List[Tuple[str, str, str, str]] = None,
         aws_profile=None,
         dir_=None,
         do_index=True,
@@ -86,48 +87,47 @@ def handle_npm_uploading(
 
         Returns the directory used for archive processing and if uploading is successful
     """
-    target_dir, valid_paths, package_metadata = _scan_metadata_paths_from_archive(
-        tarball_path, prod=product, dir__=dir_
-    )
-    if not os.path.isdir(target_dir):
-        logger.error("Error: the extracted target_dir path %s does not exist.", target_dir)
-        sys.exit(1)
+    for target in targets:
+        bucket_ = target[1]
+        prefix__ = remove_prefix(target[2], "/")
+        registry__ = target[3]
+        target_dir, valid_paths, package_metadata = _scan_metadata_paths_from_archive(
+            tarball_path, registry__, prod=product, dir__=dir_
+        )
+        if not os.path.isdir(target_dir):
+            logger.error("Error: the extracted target_dir path %s does not exist.", target_dir)
+            sys.exit(1)
+        valid_dirs = __get_path_tree(valid_paths, target_dir)
+
+        # main_target = targets[0]
+        client = S3Client(aws_profile=aws_profile, dry_run=dry_run)
+        logger.info("Start uploading files to s3 buckets: %s", bucket_)
+        failed_files = client.upload_files(
+            file_paths=valid_paths,
+            targets=[(bucket_, prefix__)],
+            product=product,
+            root=target_dir
+        )
 
-    valid_dirs = __get_path_tree(valid_paths, target_dir)
+        logger.info("Files uploading done\n")
 
-    # main_target = targets[0]
-    client = S3Client(aws_profile=aws_profile, dry_run=dry_run)
-    targets_ = [(target[1], remove_prefix(target[2], "/")) for target in targets]
-    logger.info(
-        "Start uploading files to s3 buckets: %s",
-        [target[1] for target in targets]
-    )
-    failed_files = client.upload_files(
-        file_paths=valid_paths,
-        targets=targets_,
-        product=product,
-        root=target_dir
-    )
-    logger.info("Files uploading done\n")
+        succeeded = True
 
-    succeeded = True
-    for target in targets:
         if not manifest_bucket_name:
             logger.warning(
                 'Warning: No manifest bucket is provided, will ignore the process of manifest '
                 'uploading\n')
         else:
             logger.info("Start uploading manifest to s3 bucket %s", manifest_bucket_name)
-            manifest_folder = target[1]
+            manifest_folder = bucket_
             manifest_name, manifest_full_path = write_manifest(valid_paths, target_dir, product)
+
             client.upload_manifest(
                 manifest_name, manifest_full_path,
                 manifest_folder, manifest_bucket_name
             )
             logger.info("Manifest uploading is done\n")
 
-        bucket_ = target[1]
-        prefix__ = remove_prefix(target[2], "/")
         logger.info(
             "Start generating package.json for package: %s in s3 bucket %s",
             package_metadata.name, bucket_
@@ -178,7 +178,7 @@ def handle_npm_uploading(
 def handle_npm_del(
         tarball_path: str,
         product: str,
-        targets: List[Tuple[str, str, str]] = None,
+        targets: List[Tuple[str, str, str, str]] = None,
         aws_profile=None,
         dir_=None,
         do_index=True,
@@ -381,11 +381,11 @@ def _gen_npm_package_metadata_for_del(
     return meta_files
 
 
-def _scan_metadata_paths_from_archive(path: str, prod="", dir__=None) -> Tuple[str, list,
-                                                                               NPMPackageMetadata]:
+def _scan_metadata_paths_from_archive(path: str, registry: str, prod="", dir__=None) ->\
+        Tuple[str, list, NPMPackageMetadata]:
     tmp_root = mkdtemp(prefix=f"npm-charon-{prod}-", dir=dir__)
     try:
-        _, valid_paths = extract_npm_tarball(path, tmp_root, True)
+        _, valid_paths = extract_npm_tarball(path, tmp_root, True, registry)
         if len(valid_paths) > 1:
             version = _scan_for_version(valid_paths[1])
             package = NPMPackageMetadata(version, True)
@@ -502,23 +502,14 @@ def _write_package_metadata_to_file(package_metadata: NPMPackageMetadata, root='
     final_package_metadata_path = os.path.join(root, package_metadata.name, PACKAGE_JSON)
     try:
         with open(final_package_metadata_path, mode='w', encoding='utf-8') as f:
-            dump(_del_none(package_metadata.__dict__.copy()), f)
+            dump(del_none(package_metadata.__dict__.copy()), f)
         return final_package_metadata_path
     except FileNotFoundError:
         logger.error(
             'Can not create file %s because of some missing folders', final_package_metadata_path
         )
 
 
-def _del_none(d):
-    for key, value in list(d.items()):
-        if value is None:
-            del d[key]
-        elif isinstance(value, dict):
-            _del_none(value)
-    return d
-
-
 def __get_path_tree(paths: str, prefix: str) -> Set[str]:
     valid_dirs = set()
     for f in paths:

charon/utils/archive.py

@@ -20,12 +20,14 @@
 import requests
 import tempfile
 import shutil
+import subresource_integrity
 from enum import Enum
 from json import load, JSONDecodeError, dump
 from typing import Tuple
 from zipfile import ZipFile, is_zipfile
-from charon.constants import NRRC_REGISTRY
+from charon.constants import DEFAULT_REGISTRY
 from charon.utils.files import digest, HashType
+from charon.utils.map import del_none
 
 logger = logging.getLogger(__name__)
 
@@ -44,7 +46,8 @@ def extract_zip_with_files(zf: ZipFile, target_dir: str, file_suffix: str, debug
     zf.extractall(target_dir, members=filtered)
 
 
-def extract_npm_tarball(path: str, target_dir: str, is_for_upload: bool) -> Tuple[str, list]:
+def extract_npm_tarball(path: str, target_dir: str, is_for_upload: bool, registry=DEFAULT_REGISTRY)\
+        -> Tuple[str, list]:
     """ Extract npm tarball will relocate the tgz file and metadata files.
         * Locate tar path ( e.g.: jquery/-/jquery-7.6.1.tgz or @types/jquery/-/jquery-2.2.3.tgz).
         * Locate version metadata path (e.g.: jquery/7.6.1 or @types/jquery/2.2.3).
@@ -68,7 +71,7 @@ def extract_npm_tarball(path: str, target_dir: str, is_for_upload: bool) -> Tupl
 
             if is_for_upload:
                 tgz_relative_path = "/".join([parse_paths[0], "-", _get_tgz_name(path)])
-                __write_npm_version_dist(path, f.path, version_data, tgz_relative_path)
+                __write_npm_version_dist(path, f.path, version_data, tgz_relative_path, registry)
 
                 os.makedirs(tarball_parent_path)
                 target = os.path.join(tarball_parent_path, os.path.basename(path))
@@ -87,25 +90,18 @@ def _get_tgz_name(path: str):
     return ""
 
 
-def _del_none(d):
-    for key, value in list(d.items()):
-        if value is None:
-            del d[key]
-        elif isinstance(value, dict):
-            _del_none(value)
-    return d
-
-
 def __write_npm_version_dist(path: str, version_meta_extract_path: str, version_data: dict,
-                             tgz_relative_path: str):
-    tarball_link = "".join(["https://", NRRC_REGISTRY, "/", tgz_relative_path])
-    shasum = digest(path, HashType.SHA1)
+                             tgz_relative_path: str, registry: str):
     dist = dict()
-    dist["tarball"] = tarball_link
-    dist["shasum"] = shasum
+    dist["tarball"] = "".join(["https://", registry, "/", tgz_relative_path])
+    dist["shasum"] = digest(path, HashType.SHA1)
+    with open(path, "rb") as tarball:
+        tarball_data = tarball.read()
+        integrity = subresource_integrity.render(tarball_data, ['sha512'])
+        dist["integrity"] = integrity
     version_data["dist"] = dist
     with open(version_meta_extract_path, mode='w', encoding='utf-8') as f:
-        dump(_del_none(version_data), f)
+        dump(del_none(version_data), f)
 
 
 def __parse_npm_package_version_paths(path: str) -> Tuple[dict, list]:

charon/utils/map.py

@@ -0,0 +1,7 @@
+def del_none(d):
+    for key, value in list(d.items()):
+        if value is None:
+            del d[key]
+        elif isinstance(value, dict):
+            del_none(value)
+    return d

requirements.txt

@@ -7,3 +7,4 @@ click==8.0.3
 requests==2.27.1
 ruamel.yaml==0.17.20
 defusedxml==0.7.1
+subresource-integrity==0.2

tests/base.py

@@ -50,6 +50,10 @@ def setUp(self):
     ea:
         bucket: "charon-test-ea"
         prefix: earlyaccess/all
+        
+    npm:
+        bucket: "charon-test-npm"
+        registry: "npm1.registry.redhat.com"
         """
         self.prepare_config(config_base, default_config_content)
 

tests/commons.py

@@ -101,6 +101,8 @@
     "@babel/code-frame/-/code-frame-7.15.8.tgz",
 ]
 CODE_FRAME_META = "@babel/code-frame/package.json"
+
+CODE_FRAME_7_14_5_META = "@babel/code-frame/7.14.5/package.json"
 # For npm indexes
 CODE_FRAME_7_14_5_INDEXES = [
     "@babel/code-frame/7.14.5/index.html",