Add option to decrypt function

Completed
Follow
trond.aabakken

It would be nice to have an option to not throw an exception if you try to decrypt a value with the wrong key. We have a deleted entity in our dataset with a value encrypted with an unknown key. When I restart the pipe the pipe stops on the entity with an exception like the one pasted below because the current key does not match whatever was used to encrypt the value.

"Traceback (most recent call last):",
    "  File \"/opt/venv/lib/python3.9/site-packages/cryptography/fernet.py\", line 119, in _verify_signature",
    "    h.verify(data[-32:])",
    "  File \"/opt/venv/lib/python3.9/site-packages/cryptography/hazmat/primitives/hmac.py\", line 74, in verify",
    "    ctx.verify(signature)",
    "  File \"/opt/venv/lib/python3.9/site-packages/cryptography/hazmat/backends/openssl/hmac.py\", line 75, in verify",
    "    raise InvalidSignature(\"Signature did not match digest.\")",
    "cryptography.exceptions.InvalidSignature: Signature did not match digest.",
    "",
    "During handling of the above exception, another exception occurred:",
    "Traceback (most recent call last):",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/transforms/dtl_transform.py\", line 1541, in transform",
    "    for transformed_entity in dtl.transformEntity(environment, entity):",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl.py\", line 558, in evaluateTransforms",
    "    transform_result = self.evaluateTransform(transform)",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl.py\", line 531, in evaluateTransform",
    "    result = DTL_TRANSFORMS[transform_name](self, transform_args)",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl_transforms.py\", line 38, in _transformIf",
    "    return context.evaluateTransformArgument(else_expr)",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl.py\", line 548, in evaluateTransformArgument",
    "    return self.evaluateTransform(transform_expr)",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl.py\", line 531, in evaluateTransform",
    "    result = DTL_TRANSFORMS[transform_name](self, transform_args)",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl_transforms.py\", line 34, in _transformIf",
    "    return context.evaluateTransformArgument(then_expr)",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl.py\", line 548, in evaluateTransformArgument",
    "    return self.evaluateTransform(transform_expr)",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl.py\", line 531, in evaluateTransform",
    "    result = DTL_TRANSFORMS[transform_name](self, transform_args)",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl_transforms.py\", line 32, in _transformIf",
    "    if context.evaluateBooleanExpression(condition) is True:",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl.py\", line 507, in evaluateBooleanExpression",
    "    return is_value_true(self.evaluateExpression(argument))",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl.py\", line 500, in evaluateExpression",
    "    return self.evaluateFunction(expression)",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl.py\", line 493, in evaluateFunction",
    "    return self.DTL_FUNCTIONS[function_name](self, function_args)",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl_functions_impl.py\", line 1541, in funcOr",
    "    if context.evaluateBooleanExpression(argument) is True:",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl.py\", line 507, in evaluateBooleanExpression",
    "    return is_value_true(self.evaluateExpression(argument))",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl.py\", line 500, in evaluateExpression",
    "    return self.evaluateFunction(expression)",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl.py\", line 493, in evaluateFunction",
    "    return self.DTL_FUNCTIONS[function_name](self, function_args)",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl_functions_impl.py\", line 1455, in funcGte",
    "    first = first_value(_evaluateArgument(context, arguments[0]))",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl_functions_impl.py\", line 79, in _evaluateArgument",
    "    return context.evaluateExpression(argument)",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl.py\", line 500, in evaluateExpression",
    "    return self.evaluateFunction(expression)",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl.py\", line 493, in evaluateFunction",
    "    return self.DTL_FUNCTIONS[function_name](self, function_args)",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl_functions_impl.py\", line 754, in funcInteger",
    "    values = _evaluateArgument(context, arguments[0])",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl_functions_impl.py\", line 79, in _evaluateArgument",
    "    return context.evaluateExpression(argument)",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl.py\", line 500, in evaluateExpression",
    "    return self.evaluateFunction(expression)",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl.py\", line 493, in evaluateFunction",
    "    return self.DTL_FUNCTIONS[function_name](self, function_args)",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl_functions_impl.py\", line 2582, in funcDecrypt",
    "    return unpack_object(f.decrypt(values), use_list=True)",
    "  File \"/opt/venv/lib/python3.9/site-packages/cryptography/fernet.py\", line 80, in decrypt",
    "    return self._decrypt_data(data, timestamp, time_info)",
    "  File \"/opt/venv/lib/python3.9/site-packages/cryptography/fernet.py\", line 137, in _decrypt_data",
    "    self._verify_signature(data)",
    "  File \"/opt/venv/lib/python3.9/site-packages/cryptography/fernet.py\", line 121, in _verify_signature",
    "    raise InvalidToken",
    "cryptography.fernet.InvalidToken",
    "",
    "During handling of the above exception, another exception occurred:",
    "Traceback (most recent call last):",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/task/datasynctask/datasynctask.py\", line 2862, in _run_task",
    "    full_data_set = self.do_sync()",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/task/datasynctask/datasynctask.py\", line 1986, in do_sync",
    "    raise e",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/task/datasynctask/datasynctask.py\", line 1959, in do_sync",
    "    self._do_sync_step5_handle_results()",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/task/datasynctask/datasynctask.py\", line 2557, in _do_sync_step5_handle_results",
    "    if process_batch(entity_batch):",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/task/datasynctask/datasynctask.py\", line 2533, in process_batch",
    "    _entity_batch = self._do_sync_step3_transforms(_entity_batch)",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/task/datasynctask/datasynctask.py\", line 2169, in _do_sync_step3_transforms",
    "    entity_batch = pipe.transform.transform(entity_batch, metrics=self.metrics)",
    "  File \"/opt/venv/lib/python3.9/site-packages/lake/transforms/dtl_transform.py\", line 1568, in transform",
    "    raise TransformException(\"DTL transform failed with error: %s.\\nReason: %s\" % (msg, formatted_exception),",
    "lake.node.exceptions.TransformException: DTL transform failed with error: .",
    "Reason: ['Traceback (most recent call last):\\n', '  File \"/opt/venv/lib/python3.9/site-packages/cryptography/fernet.py\", line 119, in _verify_signature\\n    h.verify(data[-32:])\\n', '  File \"/opt/venv/lib/python3.9/site-packages/cryptography/hazmat/primitives/hmac.py\", line 74, in verify\\n    ctx.verify(signature)\\n', '  File \"/opt/venv/lib/python3.9/site-packages/cryptography/hazmat/backends/openssl/hmac.py\", line 75, in verify\\n    raise InvalidSignature(\"Signature did not match digest.\")\\n', 'cryptography.exceptions.InvalidSignature: Signature did not match digest.\\n', '\\nDuring handling of the above exception, another exception occurred:\\n\\n', 'Traceback (most recent call last):\\n', '  File \"/opt/venv/lib/python3.9/site-packages/lake/transforms/dtl_transform.py\", line 1541, in transform\\n    for transformed_entity in dtl.transformEntity(environment, entity):\\n', '  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl.py\", line 558, in evaluateTransforms\\n    transform_result = self.evaluateTransform(transform)\\n', '  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl.py\", line 531, in evaluateTransform\\n    result = DTL_TRANSFORMS[transform_name](self, transform_args)\\n', '  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl_transforms.py\", line 38, in _transformIf\\n    return context.evaluateTransformArgument(else_expr)\\n', '  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl.py\", line 548, in evaluateTransformArgument\\n    return self.evaluateTransform(transform_expr)\\n', '  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl.py\", line 531, in evaluateTransform\\n    result = DTL_TRANSFORMS[transform_name](self, transform_args)\\n', '  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl_transforms.py\", line 34, in _transformIf\\n    return context.evaluateTransformArgument(then_expr)\\n', '  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl.py\", line 548, in evaluateTransformArgument\\n    return self.evaluateTransform(transform_expr)\\n', '  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl.py\", line 531, in evaluateTransform\\n    result = DTL_TRANSFORMS[transform_name](self, transform_args)\\n', '  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl_transforms.py\", line 32, in _transformIf\\n    if context.evaluateBooleanExpression(condition) is True:\\n', '  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl.py\", line 507, in evaluateBooleanExpression\\n    return is_value_true(self.evaluateExpression(argument))\\n', '  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl.py\", line 500, in evaluateExpression\\n    return self.evaluateFunction(expression)\\n', '  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl.py\", line 493, in evaluateFunction\\n    return self.DTL_FUNCTIONS[function_name](self, function_args)\\n', '  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl_functions_impl.py\", line 1541, in funcOr\\n    if context.evaluateBooleanExpression(argument) is True:\\n', '  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl.py\", line 507, in evaluateBooleanExpression\\n    return is_value_true(self.evaluateExpression(argument))\\n', '  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl.py\", line 500, in evaluateExpression\\n    return self.evaluateFunction(expression)\\n', '  File \"/opt/venv/lib/python3.9/site-packages/lake/dtl/dtl.py\", line 493, in evaluateFunction\\n    return self.DTL_FUNCTIONS[function_name](self, funct<truncated 1933 bytes>"

0

Comments

1 comment

Date Votes
  • Official comment
    Geir Ove Grønmo

    DTL functions should in general not cause the pump to fail. Most functions return null when they get invalid input, and so should this function. We'll fix this function so that it returns null instead of failing.

    Comment actions Permalink

Please sign in to leave a comment.

Didn't find what you were looking for?

New post