module-transaction: Add support for binary conversations

A module can now initiate a binary conversation decoding the native
pointer value as it wants.

Added tests to verify the main cases

cmd/pam-moduler/tests/internal/utils/test-utils.go

@@ -1,9 +1,16 @@
 // Package utils contains the internal test utils
 package utils
 
+//#include <stdint.h>
+import "C"
+
 import (
+	"crypto/rand"
+	"encoding/binary"
 	"errors"
 	"fmt"
+	"reflect"
+	"unsafe"
 
 	"github.com/msteinert/pam/v2"
 )
@@ -167,3 +174,90 @@ func (c Credentials) RespondPAM(s pam.Style, msg string) (string, error) {
 	return "", errors.Join(pam.ErrConv,
 		&SerializableError{fmt.Sprintf("unhandled style: %v", s)})
 }
+
+// BinaryTransaction represents a binary PAM transaction handler struct.
+type BinaryTransaction struct {
+	data         []byte
+	ExpectedNull bool
+	ReturnedData []byte
+}
+
+// TestBinaryDataEncoder encodes a test binary data.
+func TestBinaryDataEncoder(bytes []byte) []byte {
+	if len(bytes) > 0xff {
+		panic("Binary transaction size not supported")
+	}
+
+	if bytes == nil {
+		return bytes
+	}
+
+	data := make([]byte, 0, len(bytes)+1)
+	data = append(data, byte(len(bytes)))
+	data = append(data, bytes...)
+	return data
+}
+
+// TestBinaryDataDecoder decodes a test binary data.
+func TestBinaryDataDecoder(ptr pam.BinaryPointer) ([]byte, error) {
+	if ptr == nil {
+		return nil, nil
+	}
+
+	length := uint8(*((*C.uint8_t)(ptr)))
+	if length == 0 {
+		return []byte{}, nil
+	}
+	return C.GoBytes(unsafe.Pointer(ptr), C.int(length+1))[1:], nil
+}
+
+// NewBinaryTransactionWithData creates a new [pam.BinaryTransaction] from bytes.
+func NewBinaryTransactionWithData(data []byte, retData []byte) BinaryTransaction {
+	t := BinaryTransaction{ReturnedData: retData}
+	t.data = TestBinaryDataEncoder(data)
+	t.ExpectedNull = data == nil
+	return t
+}
+
+// NewBinaryTransactionWithRandomData creates a new [pam.BinaryTransaction] with random data.
+func NewBinaryTransactionWithRandomData(size uint8, retData []byte) BinaryTransaction {
+	t := BinaryTransaction{ReturnedData: retData}
+	randomData := make([]byte, size)
+	if err := binary.Read(rand.Reader, binary.LittleEndian, &randomData); err != nil {
+		panic(err)
+	}
+
+	t.data = TestBinaryDataEncoder(randomData)
+	return t
+}
+
+// Data returns the bytes of the transaction.
+func (b BinaryTransaction) Data() []byte {
+	return b.data
+}
+
+// RespondPAM (not) handles the PAM string conversations.
+func (b BinaryTransaction) RespondPAM(s pam.Style, msg string) (string, error) {
+	return "", errors.Join(pam.ErrConv,
+		&SerializableError{"unexpected non-binary request"})
+}
+
+// RespondPAMBinary handles the PAM binary conversations.
+func (b BinaryTransaction) RespondPAMBinary(ptr pam.BinaryPointer) ([]byte, error) {
+	if ptr == nil && !b.ExpectedNull {
+		return nil, errors.Join(pam.ErrConv,
+			&SerializableError{"unexpected null binary data"})
+	} else if ptr == nil {
+		return TestBinaryDataEncoder(b.ReturnedData), nil
+	}
+
+	bytes, _ := TestBinaryDataDecoder(ptr)
+	if !reflect.DeepEqual(bytes, b.data[1:]) {
+		return nil, errors.Join(pam.ErrConv,
+			&SerializableError{
+				fmt.Sprintf("data mismatch %#v vs %#v", bytes, b.data[1:]),
+			})
+	}
+
+	return TestBinaryDataEncoder(b.ReturnedData), nil
+}