Merge pull request #221 from ligangty/re-checksum

Add new refresh checksum cmd

Author: ligangty

charon/cmd/__init__.py

@@ -17,7 +17,7 @@
 from charon.cmd.cmd_upload import upload
 from charon.cmd.cmd_delete import delete
 from charon.cmd.cmd_index import index
-from charon.cmd.cmd_checksum import checksum_validate
+from charon.cmd.cmd_checksum import init_checksum, checksum
 from charon.cmd.cmd_cache import init_cf, cf
 
 
@@ -33,8 +33,11 @@ def cli():
 cli.add_command(upload)
 cli.add_command(delete)
 cli.add_command(index)
-cli.add_command(checksum_validate)
 
 # init cf cmmand
 init_cf()
 cli.add_command(cf)
+
+# init checksum command
+init_checksum()
+cli.add_command(checksum)

charon/cmd/cmd_cache.py

@@ -54,7 +54,7 @@
     "-f",
     "path_file",
     help="""
-    The file which contain the paths to be invalidated in CF. Pahts in this file follow the
+    The file which contain the paths to be invalidated in CF. Paths in this file follow the
     format of CF defining too, and each path should be in a single line.
     """
 )

charon/cmd/cmd_checksum.py

@@ -13,12 +13,14 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
-from typing import List
+from typing import List, Tuple
 
 from charon.config import get_config
-from charon.pkgs.checksum_http import handle_checksum_validation_http
+from charon.pkgs.checksum_http import (
+    handle_checksum_validation_http, refresh_checksum
+)
 from charon.cmd.internal import _decide_mode
-from click import command, option, argument
+from click import command, option, argument, group
 
 import traceback
 import logging
@@ -99,7 +101,7 @@
     required=True
 )
 @command()
-def checksum_validate(
+def validate(
     path: str,
     target: str,
     includes: List[str],
@@ -118,22 +120,11 @@ def checksum_validate(
     be recorded.
     """
     _decide_mode(
-        "checksum-{}".format(target), path.replace("/", "_"),
+        "checksum-validate-{}".format(target), path.replace("/", "_"),
         is_quiet=quiet, is_debug=debug
     )
     try:
-        conf = get_config()
-        if not conf:
-            sys.exit(1)
-
-        aws_bucket = ""
-        root_path = ""
-        t = conf.get_target(target)
-        if not t:
-            sys.exit(1)
-        for b in t:
-            aws_bucket = b.get('bucket')
-            prefix = b.get('prefix', '')
+        (aws_bucket, prefix) = _init_cmd(target)
 
         # NOTE: This is a liitle hacky, which constrain the configuration of
         #       of target should define the bucket to contain "prod-maven"
@@ -153,3 +144,129 @@ def checksum_validate(
     except Exception:
         print(traceback.format_exc())
         sys.exit(2)
+
+
+@option(
+    "--debug",
+    "-D",
+    "debug",
+    help="Debug mode, will print all debug logs for problem tracking.",
+    is_flag=True,
+    default=False
+)
+@option(
+    "--quiet",
+    "-q",
+    "quiet",
+    help="Quiet mode, will shrink most of the logs except warning and errors.",
+    is_flag=True,
+    default=False
+)
+@option(
+    "--path",
+    "-p",
+    "paths",
+    help="""
+    The paths of artifact files to do checksum refreshing.
+    """,
+    multiple=True
+)
+@option(
+    "--path-file",
+    "-f",
+    "path_file",
+    help="""
+    The file which contain the paths of artifact files to do checksum refreshing.
+    Each path in this file should be in a single line.
+    """
+)
+@option(
+    "--target",
+    "-t",
+    "target",
+    help="""
+    The target to do the uploading, which will decide which s3 bucket
+    and what root path where all files will be uploaded to.
+    Can accept more than one target.
+    """,
+    required=True
+)
+@command()
+def refresh(
+    target: str,
+    paths: List[str],
+    path_file: str,
+    quiet: bool = False,
+    debug: bool = False
+):
+    """
+    Refresh the checksum of the specified path for the target maven repository.
+    It will calculate the checksum files of the specified artifact and see if
+    unmatched, then regenerate the checksum files based on the artifact.
+    Default checksum files include .md5, .sha1.
+    """
+    _decide_mode(
+        "checksum-refresh-{}".format(target), "",
+        is_quiet=quiet, is_debug=debug, use_log_file=False
+    )
+    if not paths and not path_file:
+        logger.error(
+            "No path specified, please specify at least one path "
+            "through --path or --path-file.")
+        sys.exit(1)
+
+    work_paths = []
+    if paths:
+        work_paths.extend(paths)
+
+    conf = get_config()
+    aws_profile = os.getenv("AWS_PROFILE") or conf.get_aws_profile()
+    if not aws_profile:
+        logger.error("No AWS profile specified!")
+        sys.exit(1)
+
+    if path_file:
+        with open(path_file, "r", encoding="utf-8") as f:
+            for line in f.readlines():
+                work_paths.append(str(line).strip())
+    try:
+        (aws_bucket, prefix) = _init_cmd(target)
+
+        # NOTE: This is a liitle hacky, which constrain the configuration of
+        #       of target should define the bucket to contain "prod-maven"
+        #       or "stage-maven" to decide that the bucket is for maven repo
+        #       in our defined aws env for production or stage
+        if "prod-maven" not in aws_bucket and "stage-maven" not in aws_bucket:
+            logger.error("The target %s is not a maven repository.", target)
+            sys.exit(1)
+
+        refresh_checksum((aws_bucket, prefix), work_paths, aws_profile)
+    except Exception:
+        print(traceback.format_exc())
+        sys.exit(2)
+
+
+def _init_cmd(target: str) -> Tuple[str, str]:
+    conf = get_config()
+    if not conf:
+        sys.exit(1)
+    aws_bucket = ""
+    t = conf.get_target(target)
+    if not t:
+        sys.exit(1)
+    for b in t:
+        aws_bucket = b.get('bucket')
+        prefix = b.get('prefix', '')
+    return (aws_bucket, prefix)
+
+
+@group()
+def checksum():
+    """checksum commands are responsible to operate checksum files
+    of maven products operated by Charon
+    """
+
+
+def init_checksum():
+    checksum.add_command(validate)
+    checksum.add_command(refresh)

charon/pkgs/checksum_http.py

@@ -13,7 +13,8 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
-from charon.utils.files import digest
+from charon.utils.files import digest, HashType
+from charon.storage import S3Client
 from typing import Tuple, List, Dict
 from html.parser import HTMLParser
 import tempfile
@@ -36,9 +37,8 @@ def handle_checksum_validation_http(
     skips: List[str] = None
 ):
     """ Handle the checksum check for maven artifacts.
-        * target contains bucket name and prefix for the bucket, which will
-          be used to store artifacts with the prefix. See target definition
-          in Charon configuration for details.
+        * bucket contains store artifacts with the prefix. See target
+          definition in Charon configuration for details.
         * path is the root path where to start the validation in the bucket.
         * includes are the file suffixes which will decide the types of files
           to do the validation.
@@ -266,3 +266,84 @@ def _decide_root_url(bucket: str) -> str:
     if bucket.strip().startswith("stage-maven"):
         return "https://maven.stage.repository.redhat.com"
     return None
+
+
+def refresh_checksum(
+    target: Tuple[str, str],
+    paths: List[str],
+    aws_profile: str = None
+):
+    """Refresh checksum for files in a given bucket.
+       * bucket contains store artifacts with the prefix. See target
+         definition in Charon configuration for details.
+       * paths are the exact files whose checksum files will be
+         refreshed with.
+    """
+    bucket_name = target[0]
+    prefix = target[1]
+    s3_client = S3Client(aws_profile=aws_profile)
+    real_prefix = prefix if prefix.strip() != "/" else ""
+    filetype_filter = [".prodinfo", ".sha1", ".sha256", ".md5"]
+    for path in paths:
+        is_artifact = True
+        for filetype in filetype_filter:
+            if path.strip().endswith(filetype):
+                is_artifact = False
+                continue
+        if not is_artifact:
+            logger.info(
+                "%s is not an artifact file for maven products. Skipped.",
+                path
+            )
+            continue
+        s3_path = os.path.join(real_prefix, path)
+        checksums = {
+            ".md5": HashType.MD5,
+            ".sha1": HashType.SHA1,
+            ".sha256": HashType.SHA256,
+            ".sha512": HashType.SHA512
+        }
+        if s3_client.file_exists_in_bucket(bucket_name, s3_path):
+            temp_f = os.path.join(tempfile.gettempdir(), path)
+            folder = os.path.dirname(temp_f)
+            try:
+                if not os.path.exists(folder):
+                    os.makedirs(folder)
+                s3_client.download_file(bucket_name, s3_path, temp_f)
+                existed_checksum_types = []
+                for file_type in checksums:
+                    s3_checksum_path = s3_path + file_type
+                    if s3_client.file_exists_in_bucket(bucket_name, s3_checksum_path):
+                        existed_checksum_types.append(file_type)
+                if existed_checksum_types:
+                    for file_type in existed_checksum_types:
+                        checksum_path = path + file_type
+                        s3_checksum_path = s3_path + file_type
+                        hash_type = checksums[file_type]
+                        correct_checksum_c = digest(temp_f, hash_type)
+                        original_checksum_c = s3_client.read_file_content(
+                            bucket_name, s3_checksum_path
+                        )
+                        if correct_checksum_c == original_checksum_c:
+                            logger.info("Checksum %s matches, no need to refresh.", checksum_path)
+                        else:
+                            logger.info("Checksum %s does not match, refreshing...", checksum_path)
+                            s3_client.simple_upload_file(
+                                file_path=checksum_path,
+                                file_content=correct_checksum_c,
+                                target=(bucket_name, prefix),
+                                mime_type="text/plain",
+                                force=True
+                            )
+                else:
+                    logger.warning(
+                        "No valid checksum files exist for %s, Skipped."
+                        " Are you sure it is a valid maven artifact?",
+                        path
+                    )
+            finally:
+                if folder and folder != tempfile.gettempdir() and os.path.exists(folder):
+                    shutil.rmtree(folder)
+            logger.info("Checksums are refreshed for artifact %s", path)
+        else:
+            logger.warning("File %s does not exist in bucket %s", s3_path, bucket_name)

charon/storage.py

@@ -675,14 +675,15 @@ def simple_upload_file(
         self, file_path: str, file_content: str,
         target: Tuple[str, str],
         mime_type: str = None,
-        check_sum_sha1: str = None
+        check_sum_sha1: str = None,
+        force: bool = False
     ):
         """ Uploads file to s3 bucket, regardless of any extra
             information like product and version info.
-            * Warning: this will directly overwrite the files even if
-            it has lots of product info, so please be careful to use.
-            If you want to upload product artifact files, please use
-            upload_files
+            * Warning: If force is set True, it will directly overwrite
+            the files even if it has lots of product info, so please be
+            careful to use. If you want to upload product artifact files,
+            please use upload_files()
         """
         bucket = target[0]
         prefix = target[1]
@@ -705,7 +706,7 @@ def simple_upload_file(
         content_type = mime_type
         if not content_type:
             content_type = DEFAULT_MIME_TYPE
-        if not existed:
+        if not existed or force:
             f_meta = {}
             if check_sum_sha1 and check_sum_sha1.strip() != "":
                 f_meta[CHECKSUM_META_KEY] = check_sum_sha1
@@ -723,7 +724,9 @@ def simple_upload_file(
                     file_path, bucket, e
                     )
         else:
-            raise FileExistsError("Error: file %s already exists, upload is forbiden.")
+            raise FileExistsError(
+                    f"Error: file {path_key} already exists, upload is forbiden."
+                  )
 
     def delete_manifest(self, product_key: str, target: str, manifest_bucket_name: str):
         if not manifest_bucket_name:
@@ -780,6 +783,10 @@ def read_file_content(self, bucket_name: str, key: str) -> str:
         file_object = bucket.Object(key)
         return str(file_object.get()['Body'].read(), 'utf-8')
 
+    def download_file(self, bucket_name: str, key: str, file_path: str):
+        bucket = self.__get_bucket(bucket_name)
+        bucket.download_file(key, file_path)
+
     def list_folder_content(self, bucket_name: str, folder: str) -> List[str]:
         """List the content in folder in an s3 bucket. Note it's not recursive,
            which means the content only contains the items in that folder, but