rtmpdh: add an stdio.h include
[libav.git] / libavformat / rtmpdh.c
CommitLineData
acd554c1
SP
1/*
2 * RTMP Diffie-Hellmann utilities
6a433fdb
MS
3 * Copyright (c) 2009 Andrej Stepanchuk
4 * Copyright (c) 2009-2010 Howard Chu
acd554c1
SP
5 * Copyright (c) 2012 Samuel Pitoiset
6 *
7 * This file is part of Libav.
8 *
9 * Libav is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public
11 * License as published by the Free Software Foundation; either
12 * version 2.1 of the License, or (at your option) any later version.
13 *
14 * Libav is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
18 *
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with Libav; if not, write to the Free Software
21 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 */
23
24/**
25 * @file
26 * RTMP Diffie-Hellmann utilities
27 */
28
43992985
DB
29#include <stdint.h>
30#include <string.h>
31
acd554c1 32#include "config.h"
43992985
DB
33
34#include "libavutil/attributes.h"
35#include "libavutil/error.h"
36#include "libavutil/mem.h"
8337b5db 37#include "libavutil/random_seed.h"
acd554c1 38
43992985
DB
39#include "rtmpdh.h"
40
acd554c1
SP
41#define P1024 \
42 "FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD1" \
43 "29024E088A67CC74020BBEA63B139B22514A08798E3404DD" \
44 "EF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245" \
45 "E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7ED" \
46 "EE386BFB5A899FA5AE9F24117C4B1FE649286651ECE65381" \
47 "FFFFFFFFFFFFFFFF"
48
49#define Q1024 \
50 "7FFFFFFFFFFFFFFFE487ED5110B4611A62633145C06E0E68" \
51 "948127044533E63A0105DF531D89CD9128A5043CC71A026E" \
52 "F7CA8CD9E69D218D98158536F92F8A1BA7F09AB6B6A8E122" \
53 "F242DABB312F3F637A262174D31BF6B585FFAE5B7A035BF6" \
54 "F71C35FDAD44CFD2D74F9208BE258FF324943328F67329C0" \
55 "FFFFFFFFFFFFFFFF"
56
63ce9fd2
MS
57#if CONFIG_GMP || CONFIG_GCRYPT
58#if CONFIG_GMP
acd554c1
SP
59#define bn_new(bn) \
60 do { \
61 bn = av_malloc(sizeof(*bn)); \
62 if (bn) \
63 mpz_init2(bn, 1); \
64 } while (0)
65#define bn_free(bn) \
66 do { \
67 mpz_clear(bn); \
68 av_free(bn); \
69 } while (0)
70#define bn_set_word(bn, w) mpz_set_ui(bn, w)
71#define bn_cmp(a, b) mpz_cmp(a, b)
72#define bn_copy(to, from) mpz_set(to, from)
73#define bn_sub_word(bn, w) mpz_sub_ui(bn, bn, w)
74#define bn_cmp_1(bn) mpz_cmp_ui(bn, 1)
75#define bn_num_bytes(bn) (mpz_sizeinbase(bn, 2) + 7) / 8
63ce9fd2
MS
76#define bn_bn2bin(bn, buf, len) \
77 do { \
78 memset(buf, 0, len); \
79 if (bn_num_bytes(bn) <= len) \
80 mpz_export(buf, NULL, 1, 1, 0, 0, bn); \
81 } while (0)
acd554c1
SP
82#define bn_bin2bn(bn, buf, len) \
83 do { \
84 bn_new(bn); \
85 if (bn) \
63ce9fd2 86 mpz_import(bn, len, 1, 1, 0, 0, buf); \
acd554c1
SP
87 } while (0)
88#define bn_hex2bn(bn, buf, ret) \
89 do { \
90 bn_new(bn); \
91 if (bn) \
92 ret = (mpz_set_str(bn, buf, 16) == 0); \
78efc69e
MS
93 else \
94 ret = 1; \
acd554c1
SP
95 } while (0)
96#define bn_modexp(bn, y, q, p) mpz_powm(bn, y, q, p)
b2f0f37d
MS
97#define bn_random(bn, num_bits) \
98 do { \
99 int bits = num_bits; \
100 mpz_set_ui(bn, 0); \
101 for (bits = num_bits; bits > 0; bits -= 32) { \
102 mpz_mul_2exp(bn, bn, 32); \
103 mpz_add_ui(bn, bn, av_get_random_seed()); \
104 } \
105 mpz_fdiv_r_2exp(bn, bn, num_bits); \
8337b5db 106 } while (0)
acd554c1
SP
107#elif CONFIG_GCRYPT
108#define bn_new(bn) bn = gcry_mpi_new(1)
109#define bn_free(bn) gcry_mpi_release(bn)
110#define bn_set_word(bn, w) gcry_mpi_set_ui(bn, w)
111#define bn_cmp(a, b) gcry_mpi_cmp(a, b)
112#define bn_copy(to, from) gcry_mpi_set(to, from)
113#define bn_sub_word(bn, w) gcry_mpi_sub_ui(bn, bn, w)
114#define bn_cmp_1(bn) gcry_mpi_cmp_ui(bn, 1)
115#define bn_num_bytes(bn) (gcry_mpi_get_nbits(bn) + 7) / 8
116#define bn_bn2bin(bn, buf, len) gcry_mpi_print(GCRYMPI_FMT_USG, buf, len, NULL, bn)
117#define bn_bin2bn(bn, buf, len) gcry_mpi_scan(&bn, GCRYMPI_FMT_USG, buf, len, NULL)
118#define bn_hex2bn(bn, buf, ret) ret = (gcry_mpi_scan(&bn, GCRYMPI_FMT_HEX, buf, 0, 0) == 0)
119#define bn_modexp(bn, y, q, p) gcry_mpi_powm(bn, y, q, p)
e9e86d9e 120#define bn_random(bn, num_bits) gcry_mpi_randomize(bn, num_bits, GCRY_WEAK_RANDOM)
acd554c1
SP
121#endif
122
123#define MAX_BYTES 18000
124
125#define dh_new() av_malloc(sizeof(FF_DH))
126
127static FFBigNum dh_generate_key(FF_DH *dh)
128{
129 int num_bytes;
130
131 num_bytes = bn_num_bytes(dh->p) - 1;
132 if (num_bytes <= 0 || num_bytes > MAX_BYTES)
133 return NULL;
134
135 bn_new(dh->priv_key);
136 if (!dh->priv_key)
137 return NULL;
e9e86d9e 138 bn_random(dh->priv_key, 8 * num_bytes);
acd554c1
SP
139
140 bn_new(dh->pub_key);
141 if (!dh->pub_key) {
142 bn_free(dh->priv_key);
143 return NULL;
144 }
145
146 bn_modexp(dh->pub_key, dh->g, dh->priv_key, dh->p);
147
148 return dh->pub_key;
149}
150
151static int dh_compute_key(FF_DH *dh, FFBigNum pub_key_bn,
127d813b 152 uint32_t secret_key_len, uint8_t *secret_key)
acd554c1
SP
153{
154 FFBigNum k;
acd554c1
SP
155
156 bn_new(k);
157 if (!k)
158 return -1;
159
160 bn_modexp(k, pub_key_bn, dh->priv_key, dh->p);
127d813b 161 bn_bn2bin(k, secret_key, secret_key_len);
acd554c1
SP
162 bn_free(k);
163
164 /* return the length of the shared secret key like DH_compute_key */
127d813b 165 return secret_key_len;
acd554c1
SP
166}
167
168void ff_dh_free(FF_DH *dh)
169{
063f7467
MS
170 if (!dh)
171 return;
acd554c1
SP
172 bn_free(dh->p);
173 bn_free(dh->g);
174 bn_free(dh->pub_key);
175 bn_free(dh->priv_key);
176 av_free(dh);
177}
178#elif CONFIG_OPENSSL
179#define bn_new(bn) bn = BN_new()
180#define bn_free(bn) BN_free(bn)
181#define bn_set_word(bn, w) BN_set_word(bn, w)
182#define bn_cmp(a, b) BN_cmp(a, b)
183#define bn_copy(to, from) BN_copy(to, from)
184#define bn_sub_word(bn, w) BN_sub_word(bn, w)
185#define bn_cmp_1(bn) BN_cmp(bn, BN_value_one())
186#define bn_num_bytes(bn) BN_num_bytes(bn)
187#define bn_bn2bin(bn, buf, len) BN_bn2bin(bn, buf)
188#define bn_bin2bn(bn, buf, len) bn = BN_bin2bn(buf, len, 0)
189#define bn_hex2bn(bn, buf, ret) ret = BN_hex2bn(&bn, buf)
190#define bn_modexp(bn, y, q, p) \
191 do { \
192 BN_CTX *ctx = BN_CTX_new(); \
193 if (!ctx) \
194 return AVERROR(ENOMEM); \
195 if (!BN_mod_exp(bn, y, q, p, ctx)) { \
196 BN_CTX_free(ctx); \
197 return AVERROR(EINVAL); \
198 } \
199 BN_CTX_free(ctx); \
200 } while (0)
201
202#define dh_new() DH_new()
203#define dh_generate_key(dh) DH_generate_key(dh)
9f1b3050
MS
204
205static int dh_compute_key(FF_DH *dh, FFBigNum pub_key_bn,
206 uint32_t secret_key_len, uint8_t *secret_key)
207{
208 if (secret_key_len < DH_size(dh))
209 return AVERROR(EINVAL);
210 return DH_compute_key(secret_key, pub_key_bn, dh);
211}
acd554c1
SP
212
213void ff_dh_free(FF_DH *dh)
214{
063f7467
MS
215 if (!dh)
216 return;
acd554c1
SP
217 DH_free(dh);
218}
219#endif
220
221static int dh_is_valid_public_key(FFBigNum y, FFBigNum p, FFBigNum q)
222{
223 FFBigNum bn = NULL;
224 int ret = AVERROR(EINVAL);
225
226 bn_new(bn);
227 if (!bn)
228 return AVERROR(ENOMEM);
229
230 /* y must lie in [2, p - 1] */
231 bn_set_word(bn, 1);
232 if (!bn_cmp(y, bn))
233 goto fail;
234
235 /* bn = p - 2 */
236 bn_copy(bn, p);
237 bn_sub_word(bn, 1);
238 if (!bn_cmp(y, bn))
239 goto fail;
240
241 /* Verify with Sophie-Germain prime
242 *
243 * This is a nice test to make sure the public key position is calculated
244 * correctly. This test will fail in about 50% of the cases if applied to
245 * random data.
246 */
247 /* y must fulfill y^q mod p = 1 */
248 bn_modexp(bn, y, q, p);
249
250 if (bn_cmp_1(bn))
251 goto fail;
252
253 ret = 0;
254fail:
255 bn_free(bn);
256
257 return ret;
258}
259
260av_cold FF_DH *ff_dh_init(int key_len)
261{
262 FF_DH *dh;
263 int ret;
264
265 if (!(dh = dh_new()))
266 return NULL;
267
268 bn_new(dh->g);
269 if (!dh->g)
270 goto fail;
271
272 bn_hex2bn(dh->p, P1024, ret);
273 if (!ret)
274 goto fail;
275
276 bn_set_word(dh->g, 2);
277 dh->length = key_len;
278
279 return dh;
280
281fail:
282 ff_dh_free(dh);
283
284 return NULL;
285}
286
287int ff_dh_generate_public_key(FF_DH *dh)
288{
289 int ret = 0;
290
291 while (!ret) {
292 FFBigNum q1 = NULL;
293
294 if (!dh_generate_key(dh))
295 return AVERROR(EINVAL);
296
297 bn_hex2bn(q1, Q1024, ret);
298 if (!ret)
299 return AVERROR(ENOMEM);
300
301 ret = dh_is_valid_public_key(dh->pub_key, dh->p, q1);
302 bn_free(q1);
303
304 if (!ret) {
305 /* the public key is valid */
306 break;
307 }
308 }
309
310 return ret;
311}
312
313int ff_dh_write_public_key(FF_DH *dh, uint8_t *pub_key, int pub_key_len)
314{
315 int len;
316
317 /* compute the length of the public key */
318 len = bn_num_bytes(dh->pub_key);
319 if (len <= 0 || len > pub_key_len)
320 return AVERROR(EINVAL);
321
322 /* convert the public key value into big-endian form */
323 memset(pub_key, 0, pub_key_len);
324 bn_bn2bin(dh->pub_key, pub_key + pub_key_len - len, len);
325
326 return 0;
327}
328
329int ff_dh_compute_shared_secret_key(FF_DH *dh, const uint8_t *pub_key,
0508faaa
MS
330 int pub_key_len, uint8_t *secret_key,
331 int secret_key_len)
acd554c1
SP
332{
333 FFBigNum q1 = NULL, pub_key_bn = NULL;
334 int ret;
335
336 /* convert the big-endian form of the public key into a bignum */
337 bn_bin2bn(pub_key_bn, pub_key, pub_key_len);
338 if (!pub_key_bn)
339 return AVERROR(ENOMEM);
340
341 /* convert the string containing a hexadecimal number into a bignum */
342 bn_hex2bn(q1, Q1024, ret);
343 if (!ret) {
344 ret = AVERROR(ENOMEM);
345 goto fail;
346 }
347
348 /* when the public key is valid we have to compute the shared secret key */
349 if ((ret = dh_is_valid_public_key(pub_key_bn, dh->p, q1)) < 0) {
350 goto fail;
0508faaa 351 } else if ((ret = dh_compute_key(dh, pub_key_bn, secret_key_len,
acd554c1
SP
352 secret_key)) < 0) {
353 ret = AVERROR(EINVAL);
354 goto fail;
355 }
356
357fail:
358 bn_free(pub_key_bn);
359 bn_free(q1);
360
361 return ret;
362}
363
063f7467 364#ifdef TEST
704a3976
AK
365
366#include <stdio.h>
367
063f7467
MS
368static int test_random_shared_secret(void)
369{
370 FF_DH *peer1 = NULL, *peer2 = NULL;
371 int ret;
372 uint8_t pubkey1[128], pubkey2[128];
373 uint8_t sharedkey1[128], sharedkey2[128];
374
375 peer1 = ff_dh_init(1024);
376 peer2 = ff_dh_init(1024);
377 if (!peer1 || !peer2) {
378 ret = AVERROR(ENOMEM);
379 goto fail;
380 }
381 if ((ret = ff_dh_generate_public_key(peer1)) < 0)
382 goto fail;
383 if ((ret = ff_dh_generate_public_key(peer2)) < 0)
384 goto fail;
385 if ((ret = ff_dh_write_public_key(peer1, pubkey1, sizeof(pubkey1))) < 0)
386 goto fail;
387 if ((ret = ff_dh_write_public_key(peer2, pubkey2, sizeof(pubkey2))) < 0)
388 goto fail;
389 if ((ret = ff_dh_compute_shared_secret_key(peer1, pubkey2, sizeof(pubkey2),
390 sharedkey1, sizeof(sharedkey1))) < 0)
391 goto fail;
392 if ((ret = ff_dh_compute_shared_secret_key(peer2, pubkey1, sizeof(pubkey1),
393 sharedkey2, sizeof(sharedkey2))) < 0)
394 goto fail;
395 if (memcmp(sharedkey1, sharedkey2, sizeof(sharedkey1))) {
396 printf("Mismatched generated shared key\n");
397 ret = AVERROR_INVALIDDATA;
398 } else {
399 printf("Generated shared key ok\n");
400 }
401fail:
402 ff_dh_free(peer1);
403 ff_dh_free(peer2);
404 return ret;
405}
406
407static const char *private_key =
408 "976C18FCADC255B456564F74F3EEDA59D28AF6B744D743F2357BFD2404797EF896EF1A"
409 "7C1CBEAAA3AB60AF3192D189CFF3F991C9CBBFD78119FCA2181384B94011943B6D6F28"
410 "9E1B708E2D1A0C7771169293F03DA27E561F15F16F0AC9BC858C77A80FA98FD088A232"
411 "19D08BE6F165DE0B02034B18705829FAD0ACB26A5B75EF";
412static const char *public_key =
413 "F272ECF8362257C5D2C3CC2229CF9C0A03225BC109B1DBC76A68C394F256ACA3EF5F64"
414 "FC270C26382BF315C19E97A76104A716FC998A651E8610A3AE6CF65D8FAE5D3F32EEA0"
415 "0B32CB9609B494116A825D7142D17B88E3D20EDD98743DE29CF37A23A9F6A58B960591"
416 "3157D5965FCB46DDA73A1F08DD897BAE88DFE6FC937CBA";
417static const uint8_t public_key_bin[] = {
418 0xf2, 0x72, 0xec, 0xf8, 0x36, 0x22, 0x57, 0xc5, 0xd2, 0xc3, 0xcc, 0x22,
419 0x29, 0xcf, 0x9c, 0x0a, 0x03, 0x22, 0x5b, 0xc1, 0x09, 0xb1, 0xdb, 0xc7,
420 0x6a, 0x68, 0xc3, 0x94, 0xf2, 0x56, 0xac, 0xa3, 0xef, 0x5f, 0x64, 0xfc,
421 0x27, 0x0c, 0x26, 0x38, 0x2b, 0xf3, 0x15, 0xc1, 0x9e, 0x97, 0xa7, 0x61,
422 0x04, 0xa7, 0x16, 0xfc, 0x99, 0x8a, 0x65, 0x1e, 0x86, 0x10, 0xa3, 0xae,
423 0x6c, 0xf6, 0x5d, 0x8f, 0xae, 0x5d, 0x3f, 0x32, 0xee, 0xa0, 0x0b, 0x32,
424 0xcb, 0x96, 0x09, 0xb4, 0x94, 0x11, 0x6a, 0x82, 0x5d, 0x71, 0x42, 0xd1,
425 0x7b, 0x88, 0xe3, 0xd2, 0x0e, 0xdd, 0x98, 0x74, 0x3d, 0xe2, 0x9c, 0xf3,
426 0x7a, 0x23, 0xa9, 0xf6, 0xa5, 0x8b, 0x96, 0x05, 0x91, 0x31, 0x57, 0xd5,
427 0x96, 0x5f, 0xcb, 0x46, 0xdd, 0xa7, 0x3a, 0x1f, 0x08, 0xdd, 0x89, 0x7b,
428 0xae, 0x88, 0xdf, 0xe6, 0xfc, 0x93, 0x7c, 0xba
429};
430static const uint8_t peer_public_key[] = {
431 0x58, 0x66, 0x05, 0x49, 0x94, 0x23, 0x2b, 0x66, 0x52, 0x13, 0xff, 0x46,
432 0xf2, 0xb3, 0x79, 0xa9, 0xee, 0xae, 0x1a, 0x13, 0xf0, 0x71, 0x52, 0xfb,
433 0x93, 0x4e, 0xee, 0x97, 0x05, 0x73, 0x50, 0x7d, 0xaf, 0x02, 0x07, 0x72,
434 0xac, 0xdc, 0xa3, 0x95, 0x78, 0xee, 0x9a, 0x19, 0x71, 0x7e, 0x99, 0x9f,
435 0x2a, 0xd4, 0xb3, 0xe2, 0x0c, 0x1d, 0x1a, 0x78, 0x4c, 0xde, 0xf1, 0xad,
436 0xb4, 0x60, 0xa8, 0x51, 0xac, 0x71, 0xec, 0x86, 0x70, 0xa2, 0x63, 0x36,
437 0x92, 0x7c, 0xe3, 0x87, 0xee, 0xe4, 0xf1, 0x62, 0x24, 0x74, 0xb4, 0x04,
438 0xfa, 0x5c, 0xdf, 0xba, 0xfa, 0xa3, 0xc2, 0xbb, 0x62, 0x27, 0xd0, 0xf4,
439 0xe4, 0x43, 0xda, 0x8a, 0x88, 0x69, 0x60, 0xe2, 0xdb, 0x75, 0x2a, 0x98,
440 0x9d, 0xb5, 0x50, 0xe3, 0x99, 0xda, 0xe0, 0xa6, 0x14, 0xc9, 0x80, 0x12,
441 0xf9, 0x3c, 0xac, 0x06, 0x02, 0x7a, 0xde, 0x74
442};
443static const uint8_t shared_secret[] = {
444 0xb2, 0xeb, 0xcb, 0x71, 0xf3, 0x61, 0xfb, 0x5b, 0x4e, 0x5c, 0x4c, 0xcf,
445 0x5c, 0x08, 0x5f, 0x96, 0x26, 0x77, 0x1d, 0x31, 0xf1, 0xe1, 0xf7, 0x4b,
446 0x92, 0xac, 0x82, 0x2a, 0x88, 0xc7, 0x83, 0xe1, 0xc7, 0xf3, 0xd3, 0x1a,
447 0x7d, 0xc8, 0x31, 0xe3, 0x97, 0xe4, 0xec, 0x31, 0x0e, 0x8f, 0x73, 0x1a,
448 0xe4, 0xf6, 0xd8, 0xc8, 0x94, 0xff, 0xa0, 0x03, 0x84, 0x03, 0x0f, 0xa5,
449 0x30, 0x5d, 0x67, 0xe0, 0x7a, 0x3b, 0x5f, 0xed, 0x4c, 0xf5, 0xbc, 0x18,
450 0xea, 0xd4, 0x77, 0xa9, 0x07, 0xb3, 0x54, 0x0b, 0x02, 0xd9, 0xc6, 0xb8,
451 0x66, 0x5e, 0xec, 0xa4, 0xcd, 0x47, 0xed, 0xc9, 0x38, 0xc6, 0x91, 0x08,
452 0xf3, 0x85, 0x9b, 0x69, 0x16, 0x78, 0x0d, 0xb7, 0x74, 0x51, 0xaa, 0x5b,
453 0x4d, 0x74, 0xe4, 0x29, 0x2e, 0x9e, 0x8e, 0xf7, 0xe5, 0x42, 0x83, 0xb0,
454 0x65, 0xb0, 0xce, 0xc6, 0xb2, 0x8f, 0x5b, 0xb0
455};
456
457static int test_ref_data(void)
458{
459 FF_DH *dh;
460 int ret = AVERROR(ENOMEM);
461 uint8_t pubkey_test[128];
462 uint8_t sharedkey_test[128];
463
464 dh = ff_dh_init(1024);
465 if (!dh)
466 goto fail;
467 bn_hex2bn(dh->priv_key, private_key, ret);
468 if (!ret)
469 goto fail;
470 bn_hex2bn(dh->pub_key, public_key, ret);
471 if (!ret)
472 goto fail;
473 if ((ret = ff_dh_write_public_key(dh, pubkey_test, sizeof(pubkey_test))) < 0)
474 goto fail;
475 if (memcmp(pubkey_test, public_key_bin, sizeof(pubkey_test))) {
476 printf("Mismatched generated public key\n");
477 ret = AVERROR_INVALIDDATA;
478 goto fail;
479 } else {
480 printf("Generated public key ok\n");
481 }
482 if ((ret = ff_dh_compute_shared_secret_key(dh, peer_public_key, sizeof(peer_public_key),
483 sharedkey_test, sizeof(sharedkey_test))) < 0)
484 goto fail;
485 if (memcmp(shared_secret, sharedkey_test, sizeof(sharedkey_test))) {
486 printf("Mismatched generated shared key\n");
487 ret = AVERROR_INVALIDDATA;
488 } else {
489 printf("Generated shared key ok\n");
490 }
491fail:
492 ff_dh_free(dh);
493 return ret;
494}
495
496int main(void)
497{
498 if (test_random_shared_secret() < 0)
499 return 1;
500 if (test_ref_data() < 0)
501 return 1;
502 return 0;
503}
504#endif