module-transaction: Add support for binary conversations

A module can now initiate a binary conversation decoding the native
pointer value as it wants.

Added tests to verify the main cases

cmd/pam-moduler/tests/integration-tester-module/integration-tester-module.go

@@ -60,6 +60,9 @@ func (m *integrationTesterModule) handleRequest(authReq *authRequest, r *Request
 		case SerializableStringConvRequest:
 			args = append(args, reflect.ValueOf(
 				pam.NewStringConvRequest(v.Style, v.Request)))
+		case SerializableBinaryConvRequest:
+			args = append(args, reflect.ValueOf(
+				pam.NewBinaryConvRequestFromBytes(v.Request)))
 		default:
 			if arg == nil {
 				args = append(args, reflect.Zero(method.Type().In(i)))
@@ -76,6 +79,12 @@ func (m *integrationTesterModule) handleRequest(authReq *authRequest, r *Request
 		case pam.StringConvResponse:
 			res.ActionArgs = append(res.ActionArgs,
 				SerializableStringConvResponse{value.Style(), value.Response()})
+		case pam.BinaryConvResponse:
+			data, err := value.Decode(utils.TestBinaryDataDecoder)
+			if err != nil {
+				return nil, err
+			}
+			res.ActionArgs = append(res.ActionArgs, SerializableBinaryConvResponse{data})
 		case pam.Error:
 			authReq.lastError = value
 			res.ActionArgs = append(res.ActionArgs, value)

cmd/pam-moduler/tests/integration-tester-module/integration-tester-module_test.go

@@ -71,6 +71,21 @@ func ensureEnv(tx *pam.Transaction, variable string, expected string) error {
 	return nil
 }
 
+func (r *Request) toBytes(t *testing.T) []byte {
+	t.Helper()
+	bytes, err := r.GOB()
+	if err != nil {
+		t.Fatalf("error: %v", err)
+		return nil
+	}
+	return bytes
+}
+
+func (r *Request) toTransactionData(t *testing.T) []byte {
+	t.Helper()
+	return utils.TestBinaryDataEncoder(r.toBytes(t))
+}
+
 func Test_Moduler_IntegrationTesterModule(t *testing.T) {
 	t.Parallel()
 	if !pam.CheckPamHasStartConfdir() {
@@ -818,6 +833,104 @@ func Test_Moduler_IntegrationTesterModule(t *testing.T) {
 				},
 			},
 		},
+		"start-conv-binary": {
+			credentials: utils.NewBinaryTransactionWithData([]byte(
+				"\x00This is a binary data request\xC5\x00\xffYes it is!"),
+				[]byte{0x01, 0x02, 0x03, 0x05, 0x00, 0x99}),
+			checkedRequests: []checkedRequest{
+				{
+					r: NewRequest("StartConv", SerializableBinaryConvRequest{
+						utils.TestBinaryDataEncoder(
+							[]byte("\x00This is a binary data request\xC5\x00\xffYes it is!")),
+					}),
+					exp: []interface{}{SerializableBinaryConvResponse{
+						[]byte{0x01, 0x02, 0x03, 0x05, 0x00, 0x99},
+					}, nil},
+				},
+				{
+					r: NewRequest("StartBinaryConv",
+						utils.TestBinaryDataEncoder(
+							[]byte("\x00This is a binary data request\xC5\x00\xffYes it is!"))),
+					exp: []interface{}{SerializableBinaryConvResponse{
+						[]byte{0x01, 0x02, 0x03, 0x05, 0x00, 0x99},
+					}, nil},
+				},
+			},
+		},
+		"start-conv-binary-handle-failure-passed-data-mismatch": {
+			expectedError: pam.ErrConv,
+			credentials: utils.NewBinaryTransactionWithData([]byte(
+				"\x00This is a binary data request\xC5\x00\xffYes it is!"),
+				[]byte{0x01, 0x02, 0x03, 0x05, 0x00, 0x99}),
+			checkedRequests: []checkedRequest{
+				{
+					r: NewRequest("StartConv", SerializableBinaryConvRequest{
+						(&Request{"Not the expected binary data", nil}).toTransactionData(t),
+					}),
+					exp: []interface{}{nil, pam.ErrConv},
+				},
+				{
+					r: NewRequest("StartBinaryConv",
+						(&Request{"Not the expected binary data", nil}).toTransactionData(t)),
+					exp: []interface{}{nil, pam.ErrConv},
+				},
+			},
+		},
+		"start-conv-binary-handle-failure-returned-data-mismatch": {
+			expectedError: pam.ErrConv,
+			credentials: utils.NewBinaryTransactionWithRandomData(100,
+				[]byte{0x01, 0x02, 0x03, 0x05, 0x00, 0x99}),
+			checkedRequests: []checkedRequest{
+				{
+					r: NewRequest("StartConv", SerializableBinaryConvRequest{
+						(&Request{"Wrong binary data", nil}).toTransactionData(t),
+					}),
+					exp: []interface{}{nil, pam.ErrConv},
+				},
+				{
+					r: NewRequest("StartBinaryConv",
+						(&Request{"Wrong binary data", nil}).toTransactionData(t)),
+					exp: []interface{}{nil, pam.ErrConv},
+				},
+			},
+		},
+		"start-conv-binary-in-nil": {
+			credentials: utils.NewBinaryTransactionWithData(nil,
+				(&Request{"Binary data", []interface{}{true, 123, 0.5, "yay!"}}).toBytes(t)),
+			checkedRequests: []checkedRequest{
+				{
+					r: NewRequest("StartConv", SerializableBinaryConvRequest{}),
+					exp: []interface{}{SerializableBinaryConvResponse{
+						(&Request{"Binary data", []interface{}{true, 123, 0.5, "yay!"}}).toBytes(t),
+					}, nil},
+				},
+				{
+					r: NewRequest("StartBinaryConv", nil),
+					exp: []interface{}{SerializableBinaryConvResponse{
+						(&Request{"Binary data", []interface{}{true, 123, 0.5, "yay!"}}).toBytes(t),
+					}, nil},
+				},
+			},
+		},
+		"start-conv-binary-out-nil": {
+			credentials: utils.NewBinaryTransactionWithData([]byte(
+				"\x00This is a binary data request\xC5\x00\xffGimme nil!"), nil),
+			checkedRequests: []checkedRequest{
+				{
+					r: NewRequest("StartConv", SerializableBinaryConvRequest{
+						utils.TestBinaryDataEncoder(
+							[]byte("\x00This is a binary data request\xC5\x00\xffGimme nil!")),
+					}),
+					exp: []interface{}{SerializableBinaryConvResponse{}, nil},
+				},
+				{
+					r: NewRequest("StartBinaryConv",
+						utils.TestBinaryDataEncoder(
+							[]byte("\x00This is a binary data request\xC5\x00\xffGimme nil!"))),
+					exp: []interface{}{SerializableBinaryConvResponse{}, nil},
+				},
+			},
+		},
 	}
 
 	for name, tc := range tests {
@@ -831,6 +944,13 @@ func Test_Moduler_IntegrationTesterModule(t *testing.T) {
 					Args: []string{socketPath}},
 			})
 
+			switch tc.credentials.(type) {
+			case pam.BinaryConversationHandler:
+				if !pam.CheckPamHasBinaryProtocol() {
+					t.Skip("Binary protocol is not supported")
+				}
+			}
+
 			tx, err := pam.StartConfDir(name, tc.user, tc.credentials, ts.WorkDir())
 			if err != nil {
 				t.Fatalf("start #error: %v", err)
@@ -1085,6 +1205,15 @@ func Test_Moduler_IntegrationTesterModule_Authenticate(t *testing.T) {
 				exp: []interface{}{nil, pam.ErrSystem},
 			}},
 		},
+		"StartConv-Binary": {
+			expectedError: pam.ErrSystem,
+			checkedRequests: []checkedRequest{{
+				r: NewRequest("StartConv", SerializableBinaryConvRequest{
+					[]byte{0x01, 0x02, 0x03, 0x05, 0x00, 0x99},
+				}),
+				exp: []interface{}{nil, pam.ErrSystem},
+			}},
+		},
 	}
 
 	for name, tc := range tests {

cmd/pam-moduler/tests/integration-tester-module/serialization.go

@@ -36,6 +36,16 @@ type SerializableStringConvResponse struct {
 	Response string
 }
 
+// SerializableBinaryConvRequest is a serializable binary request.
+type SerializableBinaryConvRequest struct {
+	Request []byte
+}
+
+// SerializableBinaryConvResponse is a serializable binary response.
+type SerializableBinaryConvResponse struct {
+	Response []byte
+}
+
 func init() {
 	gob.Register(map[string]string{})
 	gob.Register(Request{})
@@ -49,5 +59,9 @@ func init() {
 		SerializableStringConvRequest{})
 	gob.RegisterName("main.SerializableStringConvResponse",
 		SerializableStringConvResponse{})
+	gob.RegisterName("main.SerializableBinaryConvRequest",
+		SerializableBinaryConvRequest{})
+	gob.RegisterName("main.SerializableBinaryConvResponse",
+		SerializableBinaryConvResponse{})
 	gob.Register(utils.SerializableError{})
 }

cmd/pam-moduler/tests/internal/utils/test-utils.go

@@ -1,9 +1,16 @@
 // Package utils contains the internal test utils
 package utils
 
+//#include <stdint.h>
+import "C"
+
 import (
+	"crypto/rand"
+	"encoding/binary"
 	"errors"
 	"fmt"
+	"reflect"
+	"unsafe"
 
 	"github.com/msteinert/pam/v2"
 )
@@ -167,3 +174,90 @@ func (c Credentials) RespondPAM(s pam.Style, msg string) (string, error) {
 	return "", errors.Join(pam.ErrConv,
 		&SerializableError{fmt.Sprintf("unhandled style: %v", s)})
 }
+
+// BinaryTransaction represents a binary PAM transaction handler struct.
+type BinaryTransaction struct {
+	data         []byte
+	ExpectedNull bool
+	ReturnedData []byte
+}
+
+// TestBinaryDataEncoder encodes a test binary data.
+func TestBinaryDataEncoder(bytes []byte) []byte {
+	if len(bytes) > 0xff {
+		panic("Binary transaction size not supported")
+	}
+
+	if bytes == nil {
+		return bytes
+	}
+
+	data := make([]byte, 0, len(bytes)+1)
+	data = append(data, byte(len(bytes)))
+	data = append(data, bytes...)
+	return data
+}
+
+// TestBinaryDataDecoder decodes a test binary data.
+func TestBinaryDataDecoder(ptr pam.BinaryPointer) ([]byte, error) {
+	if ptr == nil {
+		return nil, nil
+	}
+
+	length := uint8(*((*C.uint8_t)(ptr)))
+	if length == 0 {
+		return []byte{}, nil
+	}
+	return C.GoBytes(unsafe.Pointer(ptr), C.int(length+1))[1:], nil
+}
+
+// NewBinaryTransactionWithData creates a new [pam.BinaryTransaction] from bytes.
+func NewBinaryTransactionWithData(data []byte, retData []byte) BinaryTransaction {
+	t := BinaryTransaction{ReturnedData: retData}
+	t.data = TestBinaryDataEncoder(data)
+	t.ExpectedNull = data == nil
+	return t
+}
+
+// NewBinaryTransactionWithRandomData creates a new [pam.BinaryTransaction] with random data.
+func NewBinaryTransactionWithRandomData(size uint8, retData []byte) BinaryTransaction {
+	t := BinaryTransaction{ReturnedData: retData}
+	randomData := make([]byte, size)
+	if err := binary.Read(rand.Reader, binary.LittleEndian, &randomData); err != nil {
+		panic(err)
+	}
+
+	t.data = TestBinaryDataEncoder(randomData)
+	return t
+}
+
+// Data returns the bytes of the transaction.
+func (b BinaryTransaction) Data() []byte {
+	return b.data
+}
+
+// RespondPAM (not) handles the PAM string conversations.
+func (b BinaryTransaction) RespondPAM(s pam.Style, msg string) (string, error) {
+	return "", errors.Join(pam.ErrConv,
+		&SerializableError{"unexpected non-binary request"})
+}
+
+// RespondPAMBinary handles the PAM binary conversations.
+func (b BinaryTransaction) RespondPAMBinary(ptr pam.BinaryPointer) ([]byte, error) {
+	if ptr == nil && !b.ExpectedNull {
+		return nil, errors.Join(pam.ErrConv,
+			&SerializableError{"unexpected null binary data"})
+	} else if ptr == nil {
+		return TestBinaryDataEncoder(b.ReturnedData), nil
+	}
+
+	bytes, _ := TestBinaryDataDecoder(ptr)
+	if !reflect.DeepEqual(bytes, b.data[1:]) {
+		return nil, errors.Join(pam.ErrConv,
+			&SerializableError{
+				fmt.Sprintf("data mismatch %#v vs %#v", bytes, b.data[1:]),
+			})
+	}
+
+	return TestBinaryDataEncoder(b.ReturnedData), nil
+}