aboutsummaryrefslogtreecommitdiffstats
path: root/libaxl_receive_handshake.c
blob: f37fe88c486bdb759ee33cb67d9dd86d40012a56 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
/* See LICENSE file for copyright and license details. */
#include "common.h"

struct response {
	union {
		struct {
			uint8_t  status;
			uint8_t  length_of_reason; /* Only used in Failed, and not in Authenticate */
			uint16_t protocol_major_version; /* Not in Authenticate */
			uint16_t protocol_minor_version; /* Not in Authenticate */
			uint16_t length_of_addition_data;
			char additional_data[];
		};
		struct {
			uint8_t  __status__1;
			uint8_t  __length_of_reason__1;
			uint16_t __protocol_major_version__1;
			uint16_t __protocol_minor_version__1;
			uint16_t __length_of_addition_data__1;
			char reason[];
		};
		struct {
			uint8_t  __status__2;
			uint8_t  __length_of_reason__2;
			uint16_t __protocol_major_version__2;
			uint16_t __protocol_minor_version__2;
			uint16_t __length_of_addition_data__2;
			uint32_t release_number;
			uint32_t resource_id_base;
			uint32_t resource_id_mask;
			uint32_t motion_buffer_size;
			uint16_t length_of_vendor;
			uint16_t maximum_request_length;
			uint8_t number_of_roots;
			uint8_t number_of_pixmap_formats;
			uint8_t image_byte_order;
			uint8_t bitmap_format_bit_order;
			uint8_t bitmap_format_scanline_unit;
			uint8_t bitmap_format_scanline_pad;
			uint8_t min_keycode;
			uint8_t max_keycode;
			uint32_t __unused;
			char vendor[];
		};
	};
};

int
libaxl_receive_handshake(LIBAXL_CONTEXT *restrict ctx, int *restrict majorp, int *restrict minorp, char **restrict reasonp, int flags)
{
	LIBAXL_CONNECTION *conn = ctx->conn;
	char *restrict inbuf, *new;
	size_t i, j, k, n, t, len, vendor_len, out_off, in_off;
	ssize_t r;
	int read_stage = 0, saved_errno, status;
	struct response *resp;
	uint32_t xid_base, xid_mask;
	struct libaxl_screen *screen;
	struct libaxl_depth *depth;
#ifdef MSG_TRUNC
	int flag_trunc;
#endif

#ifdef MSG_TRUNC
	flags ^= flag_trunc = flags & MSG_TRUNC;
#endif

	WLOCK_CONNECTION_RECV(conn);

	inbuf = conn->in_buf;
	n = offsetof(struct response, additional_data);

continue_read:
	if (conn->in_buf_size < n) {
		inbuf = liberror_realloc(inbuf, n);
		if (!inbuf) {
			WUNLOCK_CONNECTION_RECV(conn);
			return LIBAXL_ERROR_SYSTEM;
		}
		conn->in_buf = inbuf;
		conn->in_buf_size = n;
	}

	while (conn->in_progress < n) {
		r = recv(conn->fd, &inbuf[conn->in_progress], n - conn->in_progress, flags);
		if (r <= 0) {
			liberror_save_backtrace(NULL);
			if (!r) {
				liberror_set_error("The connection to the display server has been closed",
				                   "libaxl_receive_handshake", "libaxl", LIBAXL_ERROR_CONNECTION_CLOSED);
				WUNLOCK_CONNECTION_RECV(conn);
				return LIBAXL_ERROR_CONNECTION_CLOSED;
			}
			liberror_recv_failed(conn->fd, &inbuf[conn->in_progress], n - conn->in_progress, flags, "<display server>");
			WUNLOCK_CONNECTION_RECV(conn);
			return LIBAXL_ERROR_SYSTEM;
		}
		conn->in_progress += (size_t)r;
	}

	if (read_stage == 0) {
		n += (size_t)ntohs(((struct response *)inbuf)->length_of_addition_data) * 4;
		read_stage = 1;
		goto continue_read;
	}

	conn->in_progress = 0;

#ifdef MSG_TRUNC
	if (flag_trunc) {
		WUNLOCK_CONNECTION_RECV(conn);
		return 0;
	}
#endif

	t = conn->in_buf_size;
	conn->in_buf_size = ctx->in_buf_size;
	ctx->in_buf_size = t;

	conn->in_buf = ctx->in_buf;
	ctx->in_buf = inbuf;

	WUNLOCK_CONNECTION_RECV(conn);

	resp = (struct response *)inbuf;

	if (majorp)
		*majorp = (int)ntohs(resp->protocol_major_version);
	if (minorp)
		*minorp = (int)ntohs(resp->protocol_minor_version);
	if (reasonp)
		*reasonp = NULL;

	status = (int)resp->status;
	switch (resp->status) {
	case LIBAXL_HANDSHAKE_FAILED:
		len = (size_t)ntohs(resp->length_of_reason);
		if (n < len + offsetof(struct response, reason))
			goto invalid;
		goto return_reason;

	case LIBAXL_HANDSHAKE_AUTHENTICATE:
		len = n - offsetof(struct response, reason);
	return_reason:
		if (reasonp) {
			*reasonp = liberror_malloc(len + 1);
			if (!*reasonp)
				return LIBAXL_ERROR_SYSTEM;
			memcpy(*reasonp, resp->reason, len);
			(*reasonp)[len] = '\0';
		}
		break;

	case LIBAXL_HANDSHAKE_SUCCESS:
		xid_base = ntohl(resp->resource_id_base);
		xid_mask = ntohl(resp->resource_id_mask);
		if (xid_base & xid_mask || (xid_base | xid_mask) >> 29)
			goto invalid;

		conn->xid_base = xid_base;
		for (conn->xid_shift = 0; !(xid_mask & 1); conn->xid_shift += 1)
			xid_mask >>= 1;
		conn->xid_max = xid_mask;
		conn->info.vendor_release              = ntohl(resp->release_number);
		conn->info.min_keycode                 = resp->min_keycode;
		conn->info.max_keycode                 = resp->max_keycode;
		conn->info.image_byte_order            = resp->image_byte_order;
		conn->info.motion_buffer_size          = ntohl(resp->motion_buffer_size);
		conn->info.maximum_request_length      = resp->maximum_request_length = ntohs(resp->maximum_request_length);
		conn->info.bitmap_format_bit_order     = resp->bitmap_format_bit_order;
		conn->info.bitmap_format_scanline_unit = resp->bitmap_format_scanline_unit;
		conn->info.bitmap_format_scanline_pad  = resp->bitmap_format_scanline_pad;

		/* These restricts are less restrictive than the protocol specifices,
		 * in case the they are modified later on */
		if (!xid_mask                             ||
		    conn->xid_max + 1 < UINT16_C(1) << 18 ||
		    (conn->xid_max & (conn->xid_max + 1)) ||
		    resp->min_keycode < 8                 ||
		    resp->image_byte_order > 1            ||
		    resp->maximum_request_length < 4096   ||
		    resp->bitmap_format_bit_order > 1     ||
		    resp->bitmap_format_scanline_unit % 8 ||
		    resp->bitmap_format_scanline_pad % 8  ||
		    resp->bitmap_format_scanline_unit > resp->bitmap_format_scanline_pad)
			goto invalid;

		vendor_len          = ntohs(resp->length_of_vendor);
		conn->info.nscreens = resp->number_of_roots;
		conn->info.nformats = resp->number_of_pixmap_formats;

		memmove(inbuf, resp->vendor, out_off = in_off = vendor_len);
		inbuf[out_off++] = '\0';
		out_off += (4 - out_off % 4) % 4;
		in_off += (4 - in_off % 4) % 4;
		in_off += offsetof(struct response, vendor);
		t = out_off;
		memmove(&inbuf[out_off], &inbuf[in_off], n - in_off);
		out_off += n - in_off;

		saved_errno = errno;
		new = realloc(inbuf, out_off);
		errno = saved_errno;
		if (new)
			inbuf = new;

		conn->info_buf     = inbuf;
		conn->info.vendor  = inbuf;
		conn->info.formats = (void *)&inbuf[t];
		conn->info.screens = screen = (void *)&inbuf[t + conn->info.nformats * 8];
		if ((size_t)conn->info.default_screen_number < conn->info.nscreens) {
			conn->info.default_screen = conn->info.screens;
			for (i = 0; i < (size_t)conn->info.default_screen_number; i++)
				conn->info.default_screen = libaxl_next_screen(conn->info.default_screen);
		} else {
			conn->info.default_screen = NULL;
		}

		for (i = 0; i < (size_t)conn->info.nscreens; i++) {
			screen->root                  = ntohl(screen->root);
			screen->default_colormap      = ntohl(screen->default_colormap);
			screen->white_pixel           = ntohl(screen->white_pixel);
			screen->black_pixel           = ntohl(screen->black_pixel);
			screen->current_input_masks   = ntohl(screen->current_input_masks);
			screen->width_in_pixels       = ntohs(screen->width_in_pixels);
			screen->height_in_pixels      = ntohs(screen->height_in_pixels);
			screen->width_in_millimeters  = ntohs(screen->width_in_millimeters);
			screen->height_in_millimeters = ntohs(screen->height_in_millimeters);
			screen->min_installed_maps    = ntohs(screen->min_installed_maps);
			screen->max_installed_maps    = ntohs(screen->max_installed_maps);
			screen->root_visual           = ntohs(screen->root_visual);
			depth = screen->allowed_depths;
			for (j = 0; j < (size_t)screen->number_of_allowed_depths; j++) {
				depth->number_of_visuals = ntohs(depth->number_of_visuals);
				for (k = 0; k < (size_t)depth->number_of_visuals; k++) {
					depth->visuals[k].visual_id        = ntohl(depth->visuals[k].visual_id);
					depth->visuals[k].colormap_entries = ntohs(depth->visuals[k].colormap_entries);
					depth->visuals[k].red_mask         = ntohl(depth->visuals[k].red_mask);
					depth->visuals[k].green_mask       = ntohl(depth->visuals[k].green_mask);
					depth->visuals[k].blue_mask        = ntohl(depth->visuals[k].blue_mask);
				}
				depth = (void *)(uintptr_t)&depth->visuals[depth->number_of_visuals];
			}
			screen = (void *)(uintptr_t)depth;
		}

		ctx->in_buf_size = 0;
		ctx->in_buf = NULL;
		break;

	default:
	invalid:
		liberror_set_error("Invalid handshake response received from the display server",
				   "libaxl_receive_handshake", "libaxl", LIBAXL_ERROR_INVALID_HANDSHAKE_RESPONSE);
		errno = EBADMSG;
		return LIBAXL_ERROR_INVALID_HANDSHAKE_RESPONSE;
	}

	return status;
}