initial implementation of singature for artifacts (#169)

* initial implementation of singature for artifacts

* add API doc to generate_sign function

* update README file

* use configuration file to controll artifacts needs sign

* fix some minor problem

* change suffix to exclude instead

* remove unused logger

* fix minor problem

* change charon.yml to a proper extension, change name of suffix confi

Author: shokakucarrier

.github/workflows/unittests.yaml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.6", "3.7", "3.8"]
+        python-version: ["3.7", "3.8", "3.9"]
 
     steps:
       - uses: actions/checkout@v3

README.md

@@ -14,6 +14,10 @@ future. And Ronda service will be hosted in AWS S3.
 
 See [AWS CLi V2 installation](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html#cliv2-linux-install)
 
+### [Optional] GnuPG CLI tool
+
+For artifact signing using keys already in GPG keystore, not required when using exported secret key file.
+
 ## Installation
 
 ### From git
@@ -49,7 +53,7 @@ to configure AWS access credentials.
 ### charon-upload: upload a repo to S3
 
 ```bash
-usage: charon upload $tarball --product/-p ${prod} --version/-v ${ver} [--root_path] [--ignore_patterns] [--debug]
+usage: charon upload $tarball --product/-p ${prod} --version/-v ${ver} [--root_path] [--ignore_patterns] [--debug] [--key_id] [--key_file] [--passphrase]
 ```
 
 This command will upload the repo in tarball to S3.

charon/cmd/command.py

@@ -96,6 +96,27 @@
     be extracted, when needed.
     """,
 )
+@option(
+    "--sign_keyid",
+    "-k",
+    help="""
+    GPG key fingerprint to sign artifacts, cannot work with --sign_keyfile
+    requires key already imported with gpg command --passphrase is needed.
+    """,
+)
+@option(
+    "--sign_keyfile",
+    "-K",
+    help="""
+    GPG key file path to sign artifacts, --passphrase is needed.
+    """,
+)
+@option(
+    "--passphrase",
+    help="""
+    The passphrase for GPG key, will require input if this parameter is not set.
+    """,
+)
 @option(
     "--debug",
     "-D",
@@ -120,6 +141,9 @@ def upload(
     root_path="maven-repository",
     ignore_patterns: List[str] = None,
     work_dir: str = None,
+    sign_keyid: str = None,
+    sign_keyfile: str = None,
+    passphrase: str = None,
     debug=False,
     quiet=False,
     dryrun=False
@@ -158,6 +182,9 @@ def upload(
                 buckets=buckets,
                 aws_profile=aws_profile,
                 dir_=work_dir,
+                key_id=sign_keyid,
+                key_file=sign_keyfile,
+                passphrase=passphrase,
                 dry_run=dryrun,
                 manifest_bucket_name=manifest_bucket_name
             )
@@ -178,6 +205,9 @@ def upload(
                 buckets=buckets,
                 aws_profile=aws_profile,
                 dir_=work_dir,
+                key_id=sign_keyid,
+                key_file=sign_keyfile,
+                passphrase=passphrase,
                 dry_run=dryrun,
                 manifest_bucket_name=manifest_bucket_name
             )

charon/config.py

@@ -36,6 +36,7 @@ def __init__(self, data: Dict):
         self.__aws_profile: str = data.get("aws_profile", None)
         self.__targets: Dict = data.get("targets", None)
         self.__manifest_bucket: str = data.get("manifest_bucket", None)
+        self.__ignore_signature_suffix: Dict = data.get("ignore_signature_suffix", None)
 
     def get_ignore_patterns(self) -> List[str]:
         return self.__ignore_patterns
@@ -52,6 +53,12 @@ def get_aws_profile(self) -> str:
     def get_manifest_bucket(self) -> str:
         return self.__manifest_bucket
 
+    def get_ignore_signature_suffix(self, package_type: str) -> List[str]:
+        xartifact_list: List = self.__ignore_signature_suffix.get(package_type)
+        if not xartifact_list:
+            logger.error("package type %s does not have ignore artifact config.", package_type)
+        return xartifact_list
+
 
 def get_config() -> Optional[CharonConfig]:
     config_file_path = os.path.join(os.getenv("HOME"), ".charon", CONFIG_FILE)

charon/pkgs/maven.py

@@ -15,12 +15,13 @@
 """
 from charon.utils.files import HashType
 import charon.pkgs.indexing as indexing
+import charon.pkgs.signature as signature
 from charon.utils.files import overwrite_file, digest, write_manifest
 from charon.utils.archive import extract_zip_all
 from charon.utils.strings import remove_prefix
 from charon.storage import S3Client
 from charon.pkgs.pkg_utils import upload_post_process, rollback_post_process
-from charon.config import get_template
+from charon.config import CharonConfig, get_template, get_config
 from charon.constants import (META_FILE_GEN_KEY, META_FILE_DEL_KEY,
                               META_FILE_FAILED, MAVEN_METADATA_TEMPLATE,
                               ARCHETYPE_CATALOG_TEMPLATE, ARCHETYPE_CATALOG_FILENAME,
@@ -260,6 +261,9 @@ def handle_maven_uploading(
     aws_profile=None,
     dir_=None,
     do_index=True,
+    key_id=None,
+    key_file=None,
+    passphrase=None,
     dry_run=False,
     manifest_bucket_name=None
 ) -> Tuple[str, bool]:
@@ -317,6 +321,7 @@ def handle_maven_uploading(
     )
     logger.info("Files uploading done\n")
     succeeded = True
+    generated_signs = []
     for bucket in buckets:
         # 5. Do manifest uploading
         if not manifest_bucket_name:
@@ -383,6 +388,34 @@ def handle_maven_uploading(
                 failed_metas.extend(_failed_metas)
                 logger.info("archetype-catalog.xml updating done in bucket %s\n", bucket_name)
 
+        # 10. Generate signature file if gpg ket is provided
+        if (key_id is not None or key_file is not None) and passphrase is not None:
+            conf = get_config()
+            if not conf:
+                sys.exit(1)
+            suffix_list = __get_suffix(PACKAGE_TYPE_MAVEN, conf)
+            artifacts = [s for s in valid_mvn_paths if not s.endswith(tuple(suffix_list))]
+            logger.info("Start generating signature for s3 bucket %s\n", bucket_name)
+            (_failed_metas, _generated_signs) = signature.generate_sign(
+                PACKAGE_TYPE_MAVEN, artifacts,
+                top_level, prefix,
+                s3_client, bucket_name,
+                key_id, key_file, passphrase
+            )
+            failed_metas.extend(_failed_metas)
+            generated_signs.extend(_generated_signs)
+            logger.info("Singature generation done.\n")
+
+            logger.info("Start upload singature files to s3 bucket %s\n", bucket_name)
+            _failed_metas = s3_client.upload_signatures(
+                meta_file_paths=generated_signs,
+                target=(bucket_name, prefix),
+                product=None,
+                root=top_level
+            )
+            failed_metas.extend(_failed_metas)
+            logger.info("Signature uploading done.\n")
+
         # this step generates index.html for each dir and add them to file list
         # index is similar to metadata, it will be overwritten everytime
         if do_index:
@@ -1011,6 +1044,12 @@ def _handle_error(err_msgs: List[str]):
     pass
 
 
+def __get_suffix(package_type: str, conf: CharonConfig) -> List[str]:
+    if package_type:
+        return conf.get_ignore_signature_suffix(package_type)
+    return []
+
+
 class VersionCompareKey:
     'Used as key function for version sorting'
     def __init__(self, obj):

charon/pkgs/npm.py

@@ -24,6 +24,8 @@
 from semantic_version import compare
 
 import charon.pkgs.indexing as indexing
+import charon.pkgs.signature as signature
+from charon.config import CharonConfig, get_config
 from charon.constants import META_FILE_GEN_KEY, META_FILE_DEL_KEY, PACKAGE_TYPE_NPM
 from charon.storage import S3Client
 from charon.utils.archive import extract_npm_tarball
@@ -70,6 +72,9 @@ def handle_npm_uploading(
         aws_profile=None,
         dir_=None,
         do_index=True,
+        key_id=None,
+        key_file=None,
+        passphrase=None,
         dry_run=False,
         manifest_bucket_name=None
 ) -> Tuple[str, bool]:
@@ -88,6 +93,7 @@ def handle_npm_uploading(
         Returns the directory used for archive processing and if uploading is successful
     """
     client = S3Client(aws_profile=aws_profile, dry_run=dry_run)
+    generated_signs = []
     for bucket in buckets:
         bucket_name = bucket[1]
         prefix = remove_prefix(bucket[2], "/")
@@ -160,6 +166,35 @@ def handle_npm_uploading(
             failed_metas.extend(_failed_metas)
             logger.info("package.json uploading done")
 
+        if (key_id is not None or key_file is not None) and passphrase is not None:
+            conf = get_config()
+            if not conf:
+                sys.exit(1)
+            suffix_list = __get_suffix(PACKAGE_TYPE_NPM, conf)
+            artifacts = [s for s in valid_paths if not s.endswith(tuple(suffix_list))]
+            if META_FILE_GEN_KEY in meta_files:
+                artifacts.extend(meta_files[META_FILE_GEN_KEY])
+            logger.info("Start generating signature for s3 bucket %s\n", bucket_name)
+            (_failed_metas, _generated_signs) = signature.generate_sign(
+                PACKAGE_TYPE_NPM, artifacts,
+                target_dir, prefix,
+                client, bucket_name,
+                key_id, key_file, passphrase
+            )
+            failed_metas.extend(_failed_metas)
+            generated_signs.extend(_generated_signs)
+            logger.info("Singature generation done.\n")
+
+            logger.info("Start upload singature files to s3 bucket %s\n", bucket_name)
+            _failed_metas = client.upload_signatures(
+                meta_file_paths=generated_signs,
+                target=(bucket_name, prefix),
+                product=None,
+                root=target_dir
+            )
+            failed_metas.extend(_failed_metas)
+            logger.info("Signature uploading done.\n")
+
         # this step generates index.html for each dir and add them to file list
         # index is similar to metadata, it will be overwritten everytime
         if do_index:
@@ -533,3 +568,9 @@ def __get_path_tree(paths: str, prefix: str) -> Set[str]:
         if dir_.startswith("@"):
             valid_dirs.add(dir_.split("/")[0])
     return valid_dirs
+
+
+def __get_suffix(package_type: str, conf: CharonConfig) -> List[str]:
+    if package_type:
+        return conf.get_ignore_signature_suffix(package_type)
+    return []