transaction: Add support for using raw binary pointers conversation handler

This requires the allocating function to provide a binary pointer that
will be free'd by the conversation handlers finalizers.

This is for a more advanced usage scenario where the binary conversion
may be handled manually.

app-transaction.go

@@ -35,6 +35,18 @@ type BinaryConversationHandler interface {
 	RespondPAMBinary(BinaryPointer) ([]byte, error)
 }
 
+// BinaryPointerConversationHandler is an interface for objects that can be used as
+// conversation callbacks during PAM authentication if binary protocol is going
+// to be supported.
+type BinaryPointerConversationHandler interface {
+	ConversationHandler
+	// RespondPAMBinary receives a pointer to the binary message. It's up to
+	// the receiver to parse it according to the protocol specifications.
+	// The function must return a pointer that is allocated via malloc or
+	// similar, as it's expected to be free'd by the conversation handler.
+	RespondPAMBinary(BinaryPointer) (BinaryPointer, error)
+}
+
 // ConversationFunc is an adapter to allow the use of ordinary functions as
 // conversation callbacks.
 type ConversationFunc func(Style, string) (string, error)
@@ -58,6 +70,20 @@ func (f BinaryConversationFunc) RespondPAM(Style, string) (string, error) {
 	return "", ErrConv
 }
 
+// BinaryPointerConversationFunc is an adapter to allow the use of ordinary
+// functions as binary pointer (only) conversation callbacks.
+type BinaryPointerConversationFunc func(BinaryPointer) (BinaryPointer, error)
+
+// RespondPAMBinary is a conversation callback adapter.
+func (f BinaryPointerConversationFunc) RespondPAMBinary(ptr BinaryPointer) (BinaryPointer, error) {
+	return f(ptr)
+}
+
+// RespondPAM is a dummy conversation callback adapter.
+func (f BinaryPointerConversationFunc) RespondPAM(Style, string) (string, error) {
+	return "", ErrConv
+}
+
 // _go_pam_conv_handler is a C wrapper for the conversation callback function.
 //
 //export _go_pam_conv_handler
@@ -88,6 +114,16 @@ func pamConvHandler(style Style, msg *C.char, handler ConversationHandler) (*C.c
 			return (*C.char)(C.CBytes(bytes)), success
 		}
 		handler = cb
+	case BinaryPointerConversationHandler:
+		if style == BinaryPrompt {
+			ptr, err := cb.RespondPAMBinary(BinaryPointer(msg))
+			if err != nil {
+				defer C.free(unsafe.Pointer(ptr))
+				return nil, C.int(ErrConv)
+			}
+			return (*C.char)(ptr), success
+		}
+		handler = cb
 	case ConversationHandler:
 		if style == BinaryPrompt {
 			return nil, C.int(ErrConv)
@@ -164,6 +200,12 @@ func start(service, user string, handler ConversationHandler, confDir string) (*
 			return nil, fmt.Errorf("%w: BinaryConversationHandler was used, but it is not supported by this platform",
 				ErrSystem)
 		}
+	case BinaryPointerConversationHandler:
+		if !CheckPamHasBinaryProtocol() {
+			return nil, fmt.Errorf(
+				"%w: BinaryPointerConversationHandler was used, but it is not supported by this platform",
+				ErrSystem)
+		}
 	}
 	t := &Transaction{
 		conv: &C.struct_pam_conv{},

cmd/pam-moduler/tests/integration-tester-module/integration-tester-module_test.go

@@ -949,6 +949,10 @@ func Test_Moduler_IntegrationTesterModule(t *testing.T) {
 				if !pam.CheckPamHasBinaryProtocol() {
 					t.Skip("Binary protocol is not supported")
 				}
+			case pam.BinaryPointerConversationHandler:
+				if !pam.CheckPamHasBinaryProtocol() {
+					t.Skip("Binary protocol is not supported")
+				}
 			}
 
 			tx, err := pam.StartConfDir(name, tc.user, tc.credentials, ts.WorkDir())

module-transaction_test.go

@@ -938,6 +938,145 @@ func testMockModuleTransaction(t *testing.T, mt *moduleTransaction) {
 				return mt.startConvImpl(mock, NewStringConvRequest(TextInfo, "prompt"))
 			},
 		},
+		"StartConv-Binary-with-PointerConvFunc": {
+			expectedValue: []byte{0x01, 0x02, 0x03, 0x05, 0x00, 0x95},
+			conversationHandler: BinaryPointerConversationFunc(func(ptr BinaryPointer) (BinaryPointer, error) {
+				bytes, _ := testBinaryDataDecoder(ptr)
+				expectedBinary := []byte(
+					"\x00This is a binary data request\xC5\x00\xffYes it is! From bytes pointer.")
+				if !reflect.DeepEqual(bytes, expectedBinary) {
+					return nil,
+						fmt.Errorf("%w: data mismatch %#v vs %#v", ErrConv, bytes, expectedBinary)
+				}
+				return allocateCBytes(testBinaryDataEncoder([]byte{
+					0x01, 0x02, 0x03, 0x05, 0x00, 0x95})), nil
+			}),
+			testFunc: func(mock *mockModuleTransaction) (any, error) {
+				bytes := testBinaryDataEncoder([]byte(
+					"\x00This is a binary data request\xC5\x00\xffYes it is! From bytes pointer."))
+				data, err := mt.startConvImpl(mock, NewBinaryConvRequestFromBytes(bytes))
+				if err != nil {
+					return data, err
+				}
+				resp, _ := data.(BinaryConvResponse)
+				return resp.Decode(testBinaryDataDecoder)
+			},
+		},
+		"StartConv-Binary-with-PointerConvFunc-and-allocated-data": {
+			expectedValue: []byte{0x01, 0x02, 0x03, 0x05, 0x00, 0x95},
+			conversationHandler: BinaryPointerConversationFunc(func(ptr BinaryPointer) (BinaryPointer, error) {
+				bytes, _ := testBinaryDataDecoder(ptr)
+				expectedBinary := []byte(
+					"\x00This is a binary data request\xC5\x00\xffYes it is! From pointer...")
+				if !reflect.DeepEqual(bytes, expectedBinary) {
+					return nil,
+						fmt.Errorf("%w: data mismatch %#v vs %#v", ErrConv, bytes, expectedBinary)
+				}
+				return allocateCBytes(testBinaryDataEncoder([]byte{
+					0x01, 0x02, 0x03, 0x05, 0x00, 0x95})), nil
+			}),
+			testFunc: func(mock *mockModuleTransaction) (any, error) {
+				bytes := testBinaryDataEncoder([]byte(
+					"\x00This is a binary data request\xC5\x00\xffYes it is! From pointer..."))
+				data, err := mt.startConvImpl(mock,
+					NewBinaryConvRequest(allocateCBytes(bytes), binaryPointerCBytesFinalizer))
+				if err != nil {
+					return data, err
+				}
+				resp, _ := data.(BinaryConvResponse)
+				return resp.Decode(testBinaryDataDecoder)
+			},
+		},
+		"StartConv-Binary-with-PointerConvFunc-and-allocated-data-erroring": {
+			expectedValue: nil,
+			expectedError: ErrConv,
+			conversationHandler: BinaryPointerConversationFunc(func(ptr BinaryPointer) (BinaryPointer, error) {
+				bytes, _ := testBinaryDataDecoder(ptr)
+				expectedBinary := []byte(
+					"\x00This is a binary data request\xC5\x00\xffYes it is! From pointer...")
+				if !reflect.DeepEqual(bytes, expectedBinary) {
+					return nil,
+						fmt.Errorf("%w: data mismatch %#v vs %#v", ErrConv, bytes, expectedBinary)
+				}
+				return allocateCBytes(testBinaryDataEncoder([]byte{
+					0x01, 0x02, 0x03, 0x05, 0x00, 0x95})), ErrConv
+			}),
+			testFunc: func(mock *mockModuleTransaction) (any, error) {
+				bytes := testBinaryDataEncoder([]byte(
+					"\x00This is a binary data request\xC5\x00\xffYes it is! From pointer..."))
+				data, err := mt.startConvImpl(mock,
+					NewBinaryConvRequest(allocateCBytes(bytes), binaryPointerCBytesFinalizer))
+				if err != nil {
+					return data, err
+				}
+				resp, _ := data.(BinaryConvResponse)
+				return resp.Decode(testBinaryDataDecoder)
+			},
+		},
+		"StartConv-Binary-with-PointerConvFunc-empty": {
+			expectedValue: []byte{},
+			conversationHandler: BinaryPointerConversationFunc(func(ptr BinaryPointer) (BinaryPointer, error) {
+				bytes, _ := testBinaryDataDecoder(ptr)
+				expectedBinary := []byte(
+					"\x00This is an empty binary data request\xC5\x00\xffYes it is!")
+				if !reflect.DeepEqual(bytes, expectedBinary) {
+					return nil,
+						fmt.Errorf("%w: data mismatch %#v vs %#v", ErrConv, bytes, expectedBinary)
+				}
+				return allocateCBytes(testBinaryDataEncoder([]byte{})), nil
+			}),
+			testFunc: func(mock *mockModuleTransaction) (any, error) {
+				bytes := testBinaryDataEncoder([]byte(
+					"\x00This is an empty binary data request\xC5\x00\xffYes it is!"))
+				data, err := mt.startConvImpl(mock, NewBinaryConvRequestFromBytes(bytes))
+				if err != nil {
+					return data, err
+				}
+				resp, _ := data.(BinaryConvResponse)
+				return resp.Decode(testBinaryDataDecoder)
+			},
+		},
+		"StartConv-Binary-with-PointerConvFunc-nil": {
+			expectedValue: []byte(nil),
+			conversationHandler: BinaryPointerConversationFunc(func(ptr BinaryPointer) (BinaryPointer, error) {
+				bytes, _ := testBinaryDataDecoder(ptr)
+				expectedBinary := []byte(
+					"\x00This is a nil binary data request\xC5\x00\xffYes it is!")
+				if !reflect.DeepEqual(bytes, expectedBinary) {
+					return nil,
+						fmt.Errorf("%w: data mismatch %#v vs %#v", ErrConv, bytes, expectedBinary)
+				}
+				return nil, nil
+			}),
+			testFunc: func(mock *mockModuleTransaction) (any, error) {
+				bytes := testBinaryDataEncoder([]byte(
+					"\x00This is a nil binary data request\xC5\x00\xffYes it is!"))
+				data, err := mt.startConvImpl(mock, NewBinaryConvRequestFromBytes(bytes))
+				if err != nil {
+					return data, err
+				}
+				resp, _ := data.(BinaryConvResponse)
+				return resp.Decode(testBinaryDataDecoder)
+			},
+		},
+		"StartConv-Binary-with-PointerConvFunc-error": {
+			expectedError: ErrConv,
+			conversationHandler: BinaryPointerConversationFunc(func(ptr BinaryPointer) (BinaryPointer, error) {
+				return nil, errors.New("got an error")
+			}),
+			testFunc: func(mock *mockModuleTransaction) (any, error) {
+				return mt.startConvImpl(mock, NewBinaryConvRequestFromBytes([]byte{}))
+			},
+		},
+		"StartConv-String-with-ConvPointerBinaryFunc": {
+			expectedError: ErrConv,
+			conversationHandler: BinaryPointerConversationFunc(func(ptr BinaryPointer) (BinaryPointer, error) {
+				return nil, nil
+			}),
+			testFunc: func(mock *mockModuleTransaction) (any, error) {
+				return mt.startConvImpl(mock, NewStringConvRequest(TextInfo, "prompt"))
+			},
+		},
 	}
 
 	for name, tc := range tests {

utils.go

@@ -2,6 +2,8 @@
 package pam
 
 /*
+#include <stdlib.h>
+
 #ifdef __SANITIZE_ADDRESS__
 #include <sanitizer/lsan_interface.h>
 #endif
@@ -20,6 +22,7 @@ import (
 	"os"
 	"runtime"
 	"time"
+	"unsafe"
 )
 
 func maybeDoLeakCheck() {
@@ -29,3 +32,11 @@ func maybeDoLeakCheck() {
 		C.maybe_do_leak_check()
 	}
 }
+
+func allocateCBytes(bytes []byte) BinaryPointer {
+	return BinaryPointer(C.CBytes(bytes))
+}
+
+func binaryPointerCBytesFinalizer(ptr BinaryPointer) {
+	C.free(unsafe.Pointer(ptr))
+}