Feature of adding product manifest generating and removing

Author: yma955

charon/cmd/command.py

@@ -148,6 +148,7 @@ def upload(
         npm_archive_type = detect_npm_archive(archive_path)
         product_key = f"{product}-{version}"
         prefix_ = conf.get_bucket_prefix(target)
+        manifest_bucket_name = conf.get_manifest_bucket()
         if npm_archive_type != NpmArchiveType.NOT_NPM:
             logger.info("This is a npm archive")
             tmp_dir = handle_npm_uploading(
@@ -157,7 +158,9 @@ def upload(
                 prefix=prefix_,
                 aws_profile=aws_profile,
                 dir_=work_dir,
-                dry_run=dryrun
+                dry_run=dryrun,
+                target=target,
+                manifest_bucket_name=manifest_bucket_name
             )
         else:
             ignore_patterns_list = None
@@ -175,7 +178,9 @@ def upload(
                 aws_profile=aws_profile,
                 prefix=prefix_,
                 dir_=work_dir,
-                dry_run=dryrun
+                dry_run=dryrun,
+                target=target,
+                manifest_bucket_name=manifest_bucket_name
             )
     except Exception:
         print(traceback.format_exc())
@@ -301,6 +306,7 @@ def delete(
         npm_archive_type = detect_npm_archive(archive_path)
         product_key = f"{product}-{version}"
         prefix_ = conf.get_bucket_prefix(target)
+        manifest_bucket_name = conf.get_manifest_bucket()
         if npm_archive_type != NpmArchiveType.NOT_NPM:
             logger.info("This is a npm archive")
             tmp_dir = handle_npm_del(
@@ -310,7 +316,9 @@ def delete(
                 prefix=prefix_,
                 aws_profile=aws_profile,
                 dir_=work_dir,
-                dry_run=dryrun
+                dry_run=dryrun,
+                target=target,
+                manifest_bucket_name=manifest_bucket_name
             )
         else:
             ignore_patterns_list = None
@@ -328,7 +336,9 @@ def delete(
                 aws_profile=aws_profile,
                 prefix=prefix_,
                 dir_=work_dir,
-                dry_run=dryrun
+                dry_run=dryrun,
+                target=target,
+                manifest_bucket_name=manifest_bucket_name
             )
     except Exception:
         print(traceback.format_exc())

charon/config.py

@@ -38,6 +38,7 @@ def __init__(self, data: Dict):
         self.__targets: Dict = data.get("targets", None)
         if not self.__targets or not isinstance(self.__targets, Dict):
             raise TypeError("Charon configuration is not correct: targets is invalid.")
+        self.__manifest_bucket: str = data.get("manifest_bucket", None)
 
     def get_ignore_patterns(self) -> List[str]:
         return self.__ignore_patterns
@@ -72,6 +73,9 @@ def get_bucket_prefix(self, target: str) -> str:
         prefix = remove_prefix(prefix, "/")
         return prefix
 
+    def get_manifest_bucket(self) -> str:
+        return self.__manifest_bucket
+
 
 def get_config() -> CharonConfig:
     config_file = os.path.join(os.getenv("HOME"), ".charon", CONFIG_FILE)

charon/constants.py

@@ -175,5 +175,5 @@
 '''
 
 PROD_INFO_SUFFIX = ".prodinfo"
-
+MANIFEST_SUFFIX = ".txt"
 DEFAULT_ERRORS_LOG = "errors.log"

charon/pkgs/maven.py

@@ -15,7 +15,7 @@
 """
 from charon.utils.files import HashType
 import charon.pkgs.indexing as indexing
-from charon.utils.files import overwrite_file, digest
+from charon.utils.files import overwrite_file, digest, write_manifest
 from charon.utils.archive import extract_zip_all
 from charon.utils.strings import remove_prefix
 from charon.storage import S3Client
@@ -261,7 +261,9 @@ def handle_maven_uploading(
     prefix=None,
     dir_=None,
     do_index=True,
-    dry_run=False
+    dry_run=False,
+    target=None,
+    manifest_bucket_name=None
 ) -> str:
     """ Handle the maven product release tarball uploading process.
         * repo is the location of the tarball in filesystem
@@ -302,6 +304,8 @@ def handle_maven_uploading(
         # Question: should we exit here?
 
     prefix_ = remove_prefix(prefix, "/")
+
+    manifest_name, manifest_full_path = write_manifest(valid_mvn_paths, top_level, prod_key)
     # 4. Do uploading
     logger.info("Start uploading files to s3")
     s3_client = S3Client(aws_profile=aws_profile, dry_run=dry_run)
@@ -312,7 +316,12 @@ def handle_maven_uploading(
     )
     logger.info("Files uploading done\n")
 
-    # 5. Use uploaded poms to scan s3 for metadata refreshment
+    # 5. Do manifest uploading
+    logger.info("Start uploading manifest to s3")
+    s3_client.upload_manifest(manifest_name, manifest_full_path, target, manifest_bucket_name)
+    logger.info("Manifest uploading is done\n")
+
+    # 6. Use uploaded poms to scan s3 for metadata refreshment
     logger.info("Start generating maven-metadata.xml files for all artifacts")
     meta_files = _generate_metadatas(
         s3=s3_client, bucket=bucket,
@@ -322,7 +331,7 @@ def handle_maven_uploading(
     logger.info("maven-metadata.xml files generation done\n")
 
     failed_metas = meta_files.get(META_FILE_FAILED, [])
-    # 6. Upload all maven-metadata.xml
+    # 7. Upload all maven-metadata.xml
     if META_FILE_GEN_KEY in meta_files:
         logger.info("Start updating maven-metadata.xml to s3")
         (_, _failed_metas) = s3_client.upload_metadatas(
@@ -335,7 +344,7 @@ def handle_maven_uploading(
         failed_metas.extend(_failed_metas)
         logger.info("maven-metadata.xml updating done\n")
 
-    # 7. Determine refreshment of archetype-catalog.xml
+    # 8. Determine refreshment of archetype-catalog.xml
     if os.path.exists(os.path.join(top_level, "archetype-catalog.xml")):
         logger.info("Start generating archetype-catalog.xml")
         upload_archetype_file = _generate_upload_archetype_catalog(
@@ -345,7 +354,7 @@ def handle_maven_uploading(
         )
         logger.info("archetype-catalog.xml files generation done\n")
 
-        # 8. Upload archetype-catalog.xml if it has changed
+        # 9. Upload archetype-catalog.xml if it has changed
         if upload_archetype_file:
             archetype_files = [os.path.join(top_level, ARCHETYPE_CATALOG_FILENAME)]
             archetype_files.extend(__hash_decorate_metadata(top_level, ARCHETYPE_CATALOG_FILENAME))
@@ -397,7 +406,9 @@ def handle_maven_del(
     prefix=None,
     dir_=None,
     do_index=True,
-    dry_run=False
+    dry_run=False,
+    target=None,
+    manifest_bucket_name=None
 ) -> str:
     """ Handle the maven product release tarball deletion process.
         * repo is the location of the tarball in filesystem
@@ -425,7 +436,6 @@ def handle_maven_del(
      valid_dirs) = _scan_paths(tmp_root, ignore_patterns, root)
 
     # 3. Delete all valid_paths from s3
-    logger.info("Start generating maven-metadata.xml files for all artifacts")
     logger.debug("Valid poms: %s", valid_poms)
     prefix_ = remove_prefix(prefix, "/")
     logger.info("Start deleting files from s3")
@@ -440,7 +450,12 @@ def handle_maven_del(
     )
     logger.info("Files deletion done\n")
 
-    # 4. Use changed GA to scan s3 for metadata refreshment
+    # 4. Delete related manifest from s3
+    logger.info("Start deleting manifest from s3")
+    s3_client.delete_manifest(prod_key, target, manifest_bucket_name)
+    logger.info("Manifest deletion is done\n")
+
+    # 5. Use changed GA to scan s3 for metadata refreshment
     logger.info("Start generating maven-metadata.xml files for all changed GAs")
     meta_files = _generate_metadatas(
         s3=s3_client, bucket=bucket,
@@ -450,7 +465,7 @@ def handle_maven_del(
 
     logger.info("maven-metadata.xml files generation done\n")
 
-    # 5. Upload all maven-metadata.xml. We need to delete metadata files
+    # 6. Upload all maven-metadata.xml. We need to delete metadata files
     # firstly for all affected GA, and then replace the theirs content.
     logger.info("Start updating maven-metadata.xml to s3")
     all_meta_files = []
@@ -475,7 +490,7 @@ def handle_maven_del(
         failed_metas.extend(_failed_metas)
     logger.info("maven-metadata.xml updating done\n")
 
-    # 6. Determine refreshment of archetype-catalog.xml
+    # 7. Determine refreshment of archetype-catalog.xml
     if os.path.exists(os.path.join(top_level, "archetype-catalog.xml")):
         logger.info("Start generating archetype-catalog.xml")
         archetype_action = _generate_rollback_archetype_catalog(
@@ -485,7 +500,7 @@ def handle_maven_del(
         )
         logger.info("archetype-catalog.xml files generation done\n")
 
-        # 7. Upload or Delete archetype-catalog.xml if it has changed
+        # 8. Upload or Delete archetype-catalog.xml if it has changed
         archetype_files = [os.path.join(top_level, ARCHETYPE_CATALOG_FILENAME)]
         archetype_files.extend(__hash_decorate_metadata(top_level, ARCHETYPE_CATALOG_FILENAME))
         if archetype_action < 0:

charon/pkgs/npm.py

@@ -29,6 +29,7 @@
 from charon.utils.archive import extract_npm_tarball
 from charon.pkgs.pkg_utils import upload_post_process, rollback_post_process
 from charon.utils.strings import remove_prefix
+from charon.utils.files import write_manifest
 
 logger = logging.getLogger(__name__)
 
@@ -62,11 +63,16 @@ def __init__(self, metadata, is_version):
 
 
 def handle_npm_uploading(
-        tarball_path: str, product: str,
-        bucket_name=None, prefix=None,
+        tarball_path: str,
+        product: str,
+        bucket_name=None,
+        prefix=None,
         aws_profile=None,
-        dir_=None, do_index=True,
-        dry_run=False
+        dir_=None,
+        do_index=True,
+        dry_run=False,
+        target=None,
+        manifest_bucket_name=None
 ) -> str:
     """ Handle the npm product release tarball uploading process.
         For NPM uploading, tgz file and version metadata will be relocated based
@@ -90,6 +96,9 @@ def handle_npm_uploading(
     valid_dirs = __get_path_tree(valid_paths, target_dir)
 
     prefix_ = remove_prefix(prefix, "/")
+
+    manifest_name, manifest_full_path = write_manifest(valid_paths, target_dir, product)
+
     logger.info("Start uploading files to s3")
     client = S3Client(aws_profile=aws_profile, dry_run=dry_run)
     bucket = bucket_name
@@ -102,6 +111,10 @@ def handle_npm_uploading(
     )
     logger.info("Files uploading done\n")
 
+    logger.info("Start uploading manifest to s3")
+    client.upload_manifest(manifest_name, manifest_full_path, target, manifest_bucket_name)
+    logger.info("Manifest uploading is done\n")
+
     logger.info("Start generating package.json for package: %s", package_metadata.name)
     meta_files = _gen_npm_package_metadata_for_upload(
         client, bucket, target_dir, package_metadata, prefix_
@@ -145,10 +158,16 @@ def handle_npm_uploading(
 
 
 def handle_npm_del(
-        tarball_path: str, product: str,
-        bucket_name=None, prefix=None,
-        aws_profile=None, dir_=None,
-        do_index=True, dry_run=False
+        tarball_path: str,
+        product: str,
+        bucket_name=None,
+        prefix=None,
+        aws_profile=None,
+        dir_=None,
+        do_index=True,
+        dry_run=False,
+        target=None,
+        manifest_bucket_name=None
 ) -> str:
     """ Handle the npm product release tarball deletion process.
         * tarball_path is the location of the tarball in filesystem
@@ -177,6 +196,10 @@ def handle_npm_del(
     )
     logger.info("Files deletion done\n")
 
+    logger.info("Start deleting manifest from s3")
+    client.delete_manifest(product, target, manifest_bucket_name)
+    logger.info("Manifest deletion is done\n")
+
     logger.info("Start generating package.json for package: %s", package_name_path)
     meta_files = _gen_npm_package_metadata_for_del(
         client, bucket, target_dir, package_name_path, prefix_
@@ -252,7 +275,7 @@ def _gen_npm_package_metadata_for_upload(
     if prefix and prefix != "/":
         package_metadata_key = os.path.join(prefix, package_metadata_key)
     (package_json_files, success) = client.get_files(bucket_name=bucket,
-                                                     prefix=package_metadata_key)
+        prefix=package_metadata_key)
     if not success:
         logger.warning("Error to get remote metadata files for %s", package_metadata_key)
     result = source_package
@@ -320,7 +343,7 @@ def _gen_npm_package_metadata_for_del(
 
 
 def _scan_metadata_paths_from_archive(path: str, prod="", dir__=None) -> Tuple[
-        str, list, NPMPackageMetadata]:
+    str, list, NPMPackageMetadata]:
     tmp_root = mkdtemp(prefix=f"npm-charon-{prod}-", dir=dir__)
     try:
         _, valid_paths = extract_npm_tarball(path, tmp_root, True)

charon/storage.py

@@ -14,9 +14,11 @@
 limitations under the License.
 """
 import asyncio
+
+from boto3.exceptions import S3UploadFailedError
 from boto3_type_annotations.s3.service_resource import Object
 from charon.utils.files import read_sha1
-from charon.constants import PROD_INFO_SUFFIX
+from charon.constants import PROD_INFO_SUFFIX, MANIFEST_SUFFIX
 
 from boto3 import session
 from botocore.errorfactory import ClientError
@@ -301,6 +303,31 @@ async def path_upload_handler(
             file_paths=meta_file_paths, path_handler=path_upload_handler, root=root
         ))
 
+    def upload_manifest(
+            self, manifest_name: str, manifest_full_path: str, target: str,
+            manifest_bucket_name: str
+    ):
+        target = target if target else "default"
+        env_folder = "-".join([target, "charon-metadata"])
+        path_key = os.path.join(env_folder, manifest_name)
+        if not manifest_bucket_name:
+            logger.warning(
+                'Warning: No manifest bucket provided, will ignore the process of manifest '
+                'uploading')
+            return
+
+        manifest_bucket = self.__client.Bucket(manifest_bucket_name)
+        try:
+            file_object: s3.Object = manifest_bucket.Object(path_key)
+            file_object.upload_file(
+                Filename=manifest_full_path,
+                ExtraArgs={'ContentType': DEFAULT_MIME_TYPE}
+            )
+        except S3UploadFailedError:
+            logger.warning(
+                'Warning: Manifest bucket %s does not exist in S3, will ignore uploading of '
+                'manifest file %s', manifest_bucket_name, manifest_name)
+
     def delete_files(
         self, file_paths: List[str], bucket_name: str,
         product: Optional[str], root="/", key_prefix: str = None
@@ -398,6 +425,26 @@ async def path_delete_handler(
 
         return (deleted_files, failed_files)
 
+    def delete_manifest(self, product_key: str, target: str, manifest_bucket_name: str):
+        manifest_name = product_key + MANIFEST_SUFFIX
+        target = target if target else "default"
+        env_folder = "-".join([target, "charon-metadata"])
+        path_key = os.path.join(env_folder, manifest_name)
+        if not manifest_bucket_name:
+            logger.warning(
+                'Warning: No manifest bucket provided, will ignore the process of manifest '
+                'deleting')
+            return
+
+        manifest_bucket = self.__client.Bucket(manifest_bucket_name)
+        file_object: s3.Object = manifest_bucket.Object(path_key)
+        if self.__file_exists(file_object):
+            manifest_bucket.delete_objects(Delete={"Objects": [{"Key": path_key}]})
+        else:
+            logger.warning(
+                'Warning: Manifest %s does not exist in S3 bucket %s, will ignore its deleting',
+                manifest_name, manifest_bucket_name)
+
     def get_files(self, bucket_name: str, prefix=None, suffix=None) -> Tuple[List[str], bool]:
         """Get the file names from s3 bucket. Can use prefix and suffix to filter the
         files wanted. If some error happend, will return an empty file list and false result