diff --git a/src/sagemaker/model_monitor/clarify_model_monitoring.py b/src/sagemaker/model_monitor/clarify_model_monitoring.py index 77f27b37f0..3edfabc747 100644 --- a/src/sagemaker/model_monitor/clarify_model_monitoring.py +++ b/src/sagemaker/model_monitor/clarify_model_monitoring.py @@ -669,6 +669,7 @@ def create_monitoring_schedule( self.monitoring_schedule_name = monitor_schedule_name except Exception: logger.exception("Failed to create monitoring schedule.") + self.monitoring_schedule_name = None # noinspection PyBroadException try: self.sagemaker_session.sagemaker_client.delete_model_bias_job_definition( @@ -1109,6 +1110,7 @@ def create_monitoring_schedule( self.monitoring_schedule_name = monitor_schedule_name except Exception: logger.exception("Failed to create monitoring schedule.") + self.monitoring_schedule_name = None # noinspection PyBroadException try: self.sagemaker_session.sagemaker_client.delete_model_explainability_job_definition( diff --git a/src/sagemaker/model_monitor/model_monitoring.py b/src/sagemaker/model_monitor/model_monitoring.py index 2800082df4..c313efcf5e 100644 --- a/src/sagemaker/model_monitor/model_monitoring.py +++ b/src/sagemaker/model_monitor/model_monitoring.py @@ -415,30 +415,34 @@ def create_monitoring_schedule( if arguments is not None: self.arguments = arguments - self.sagemaker_session.create_monitoring_schedule( - monitoring_schedule_name=self.monitoring_schedule_name, - schedule_expression=schedule_cron_expression, - statistics_s3_uri=statistics_s3_uri, - constraints_s3_uri=constraints_s3_uri, - monitoring_inputs=[normalized_monitoring_input], - monitoring_output_config=monitoring_output_config, - instance_count=self.instance_count, - instance_type=self.instance_type, - volume_size_in_gb=self.volume_size_in_gb, - volume_kms_key=self.volume_kms_key, - image_uri=self.image_uri, - entrypoint=self.entrypoint, - arguments=self.arguments, - record_preprocessor_source_uri=None, - post_analytics_processor_source_uri=None, - max_runtime_in_seconds=self.max_runtime_in_seconds, - environment=self.env, - network_config=network_config_dict, - role_arn=self.sagemaker_session.expand_role(self.role), - tags=self.tags, - data_analysis_start_time=data_analysis_start_time, - data_analysis_end_time=data_analysis_end_time, - ) + try: + self.sagemaker_session.create_monitoring_schedule( + monitoring_schedule_name=self.monitoring_schedule_name, + schedule_expression=schedule_cron_expression, + statistics_s3_uri=statistics_s3_uri, + constraints_s3_uri=constraints_s3_uri, + monitoring_inputs=[normalized_monitoring_input], + monitoring_output_config=monitoring_output_config, + instance_count=self.instance_count, + instance_type=self.instance_type, + volume_size_in_gb=self.volume_size_in_gb, + volume_kms_key=self.volume_kms_key, + image_uri=self.image_uri, + entrypoint=self.entrypoint, + arguments=self.arguments, + record_preprocessor_source_uri=None, + post_analytics_processor_source_uri=None, + max_runtime_in_seconds=self.max_runtime_in_seconds, + environment=self.env, + network_config=network_config_dict, + role_arn=self.sagemaker_session.expand_role(self.role), + tags=self.tags, + data_analysis_start_time=data_analysis_start_time, + data_analysis_end_time=data_analysis_end_time, + ) + except Exception: + self.monitoring_schedule_name = None + raise def update_monitoring_schedule( self, @@ -2054,6 +2058,7 @@ def create_monitoring_schedule( self.monitoring_schedule_name = monitor_schedule_name except Exception: logger.exception("Failed to create monitoring schedule.") + self.monitoring_schedule_name = None # noinspection PyBroadException try: self.sagemaker_session.sagemaker_client.delete_data_quality_job_definition( @@ -3173,6 +3178,7 @@ def create_monitoring_schedule( self.monitoring_schedule_name = monitor_schedule_name except Exception: logger.exception("Failed to create monitoring schedule.") + self.monitoring_schedule_name = None # noinspection PyBroadException try: self.sagemaker_session.sagemaker_client.delete_model_quality_job_definition( diff --git a/tests/integ/test_model_monitor.py b/tests/integ/test_model_monitor.py index 1a11b08f7d..17ea70699b 100644 --- a/tests/integ/test_model_monitor.py +++ b/tests/integ/test_model_monitor.py @@ -2488,3 +2488,85 @@ def test_one_time_monitoring_schedule(sagemaker_session): my_default_monitor.stop_monitoring_schedule() my_default_monitor.delete_monitoring_schedule() raise e + + +def test_create_monitoring_schedule_with_validation_error(sagemaker_session): + my_default_monitor = DefaultModelMonitor( + role=ROLE, + instance_count=INSTANCE_COUNT, + instance_type=INSTANCE_TYPE, + volume_size_in_gb=VOLUME_SIZE_IN_GB, + max_runtime_in_seconds=MAX_RUNTIME_IN_SECONDS, + sagemaker_session=sagemaker_session, + env=ENVIRONMENT, + tags=TAGS, + network_config=NETWORK_CONFIG, + ) + + output_s3_uri = os.path.join( + "s3://", + sagemaker_session.default_bucket(), + "integ-test-monitoring-output-bucket", + str(uuid.uuid4()), + ) + + data_captured_destination_s3_uri = os.path.join( + "s3://", + sagemaker_session.default_bucket(), + "sagemaker-serving-batch-transform", + str(uuid.uuid4()), + ) + + batch_transform_input = BatchTransformInput( + data_captured_destination_s3_uri=data_captured_destination_s3_uri, + destination="/opt/ml/processing/output", + dataset_format=MonitoringDatasetFormat.csv(header=False), + ) + + statistics = Statistics.from_file_path( + statistics_file_path=os.path.join(tests.integ.DATA_DIR, "monitor/statistics.json"), + sagemaker_session=sagemaker_session, + ) + + constraints = Constraints.from_file_path( + constraints_file_path=os.path.join(tests.integ.DATA_DIR, "monitor/constraints.json"), + sagemaker_session=sagemaker_session, + ) + + try: + my_default_monitor.create_monitoring_schedule( + monitor_schedule_name="schedule-name-more-than-63-characters-to-get-a-validation-exception", + batch_transform_input=batch_transform_input, + output_s3_uri=output_s3_uri, + statistics=statistics, + constraints=constraints, + schedule_cron_expression=CronExpressionGenerator.now(), + data_analysis_start_time="-PT1H", + data_analysis_end_time="-PT0H", + enable_cloudwatch_metrics=ENABLE_CLOUDWATCH_METRICS, + ) + except Exception as e: + assert "ValidationException" in str(e) + + my_default_monitor.create_monitoring_schedule( + monitor_schedule_name=unique_name_from_base("valid-schedule-name"), + batch_transform_input=batch_transform_input, + output_s3_uri=output_s3_uri, + statistics=statistics, + constraints=constraints, + schedule_cron_expression=CronExpressionGenerator.now(), + data_analysis_start_time="-PT1H", + data_analysis_end_time="-PT0H", + enable_cloudwatch_metrics=ENABLE_CLOUDWATCH_METRICS, + ) + try: + + _wait_for_schedule_changes_to_apply(monitor=my_default_monitor) + + my_default_monitor.stop_monitoring_schedule() + my_default_monitor.delete_monitoring_schedule() + + except Exception as e: + my_default_monitor.stop_monitoring_schedule() + my_default_monitor.delete_monitoring_schedule() + raise e