Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .mise.toml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
[tools]
python="3.11"
poetry="2.2"
poetry="2.2.1"
java="liberica-1.8.0"
2 changes: 1 addition & 1 deletion .tool-versions
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
python 3.11.14
poetry 2.2.0
poetry 2.2.1
java liberica-1.8.0
8 changes: 8 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,11 @@
## v0.6.1 (2026-02-19)

### Fix

- ensure that captured errors during business rule evaluation are being captured and logged
- included submission status (with additional processing failure check) in error report population to reduce chance of incorrect status
- issue with si filename handling when the filename contains special chars

## v0.6.0 (2026-02-16)

### Feat
Expand Down
21 changes: 7 additions & 14 deletions poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "nhs_dve"
version = "0.6.0"
version = "0.6.1"
description = "`nhs data validation engine` is a framework used to validate data"
authors = ["NHS England <[email protected]>"]
readme = "README.md"
Expand Down Expand Up @@ -51,7 +51,7 @@ faker = "18.11.1"
behave = "1.3.3"
coverage = "7.11.0"
moto = {extras = ["s3"], version = "4.0.13"}
Werkzeug = "3.0.6" # Dependency of moto which needs 3.0.6 for security vuln mitigation
Werkzeug = "3.1.5"
pytest = "8.4.2"
pytest-lazy-fixtures = "1.4.0" # switched from https://github.com/TvoroG/pytest-lazy-fixture as it's no longer supported
xlsx2csv = "0.8.2"
Expand Down
31 changes: 23 additions & 8 deletions src/dve/core_engine/backends/base/rules.py
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ def _step_metadata_to_location(step_metadata: "AbstractStep") -> str:

def _handle_rule_error(self, error: Exception, config: AbstractStep) -> Messages:
"""Log an error and create appropriate error messages."""
return render_error(error, self._step_metadata_to_location(config))
return render_error(error, self._step_metadata_to_location(config), self.logger)

def evaluate(self, entities, *, config: AbstractStep) -> tuple[Messages, StageSuccessful]:
"""Evaluate a step definition, applying it to the entities."""
Expand Down Expand Up @@ -411,7 +411,7 @@ def apply_sync_filters(
CriticalProcessingError(
"Issue occurred while applying filter logic",
messages=[
msg.error_message
msg.error_message # type: ignore
for msg in temp_messages
if msg.error_message
],
Expand Down Expand Up @@ -439,7 +439,10 @@ def apply_sync_filters(
[
CriticalProcessingError(
"Issue occurred while generating FeedbackMessages",
[msg.error_message for msg in temp_messages],
messages=[
msg.error_message # type: ignore
for msg in temp_messages
],
)
],
)
Expand Down Expand Up @@ -467,7 +470,10 @@ def apply_sync_filters(
[
CriticalProcessingError(
"Issue occurred while generating FeedbackMessages",
[msg.error_message for msg in temp_messages],
messages=[
msg.error_message # type: ignore
for msg in temp_messages
],
)
],
)
Expand Down Expand Up @@ -505,7 +511,9 @@ def apply_sync_filters(
[
CriticalProcessingError(
"Issue occurred while filtering error records",
[msg.error_message for msg in temp_messages],
messages=[
msg.error_message for msg in temp_messages # type: ignore
],
)
],
)
Expand Down Expand Up @@ -533,7 +541,10 @@ def apply_sync_filters(
[
CriticalProcessingError(
"Issue occurred while generating FeedbackMessages",
[msg.error_message for msg in temp_messages],
messages=[
msg.error_message # type: ignore
for msg in temp_messages
],
)
],
)
Expand Down Expand Up @@ -592,7 +603,9 @@ def apply_rules(
[
CriticalProcessingError(
"Issue occurred while applying pre filter steps",
[msg.error_message for msg in stage_messages],
messages=[
msg.error_message for msg in stage_messages # type: ignore
],
)
],
)
Expand Down Expand Up @@ -644,7 +657,9 @@ def apply_rules(
[
CriticalProcessingError(
"Issue occurred while applying post filter steps",
[msg.error_message for msg in stage_messages],
messages=[
msg.error_message for msg in stage_messages # type: ignore
],
)
],
)
Expand Down
11 changes: 2 additions & 9 deletions src/dve/core_engine/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import os
import uuid
from collections.abc import MutableMapping
from pathlib import Path, PurePath
from pathlib import Path
from typing import Any, Optional

from pydantic import UUID4, BaseModel, Field, FilePath, root_validator, validator
Expand Down Expand Up @@ -64,16 +64,9 @@ class SubmissionInfo(AuditRecord):
datetime_received: Optional[dt.datetime] = None # type: ignore
"""The datetime the file was received."""

@validator("file_name")
def _ensure_metadata_extension_removed(cls, filename): # pylint: disable=no-self-argument
path = PurePath(filename)
return path.stem

@validator("file_extension")
def _ensure_just_file_stem(cls, extension: str): # pylint: disable=no-self-argument
if "." in extension:
return extension.split(".")[-1]
return extension
return extension.rsplit(".", 1)[-1]

@property
def file_name_with_ext(self):
Expand Down
9 changes: 9 additions & 0 deletions src/dve/pipeline/pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
dump_feedback_errors,
dump_processing_errors,
get_feedback_errors_uri,
get_processing_errors_uri,
load_feedback_messages,
)
from dve.core_engine.backends.base.auditing import BaseAuditingManager
Expand Down Expand Up @@ -769,6 +770,13 @@ def error_report(
"error_report", submission_info.submission_id
)

if not submission_status.processing_failed:
submission_status.processing_failed = fh.get_resource_exists(
get_processing_errors_uri(
fh.joinuri(self.processed_files_path, submission_info.submission_id)
)
)

if not self.processed_files_path:
raise AttributeError("processed files path not provided")

Expand Down Expand Up @@ -797,6 +805,7 @@ def error_report(
if value is not None and not key.endswith("_updated")
}
summary_items = er.SummaryItems(
submission_status=submission_status,
summary_dict=summary_dict,
row_headings=["Submission Failure", "Warning"],
)
Expand Down
Loading